## import models and tokenisers

from transformers import AutoTokenizer, AutoModel
# AutoTokenizer automatically retrieves the model's configuration, pre-trained weights, vocabulary, etc.

model_names = ["bert-base-uncased",
               "Musixmatch/umberto-commoncrawl-cased-v1",
                "Musixmatch/umberto-wikipedia-uncased-v1",
                "idb-ita/gilberto-uncased-from-camembert",
                "m-polignano-uniba/bert_uncased_L-12_H-768_A-12_italian_alb3rt0"]

simple_names = ["BERT",
                "UmBERToCC",
                "UmBERToW",
                "GilBERTo",
                "AlBERTo"]

# store (name,tokeniser,model) tuples
model_pairs = [] 
m = 0

for name in model_names:
    model_name = name

    tokenizer = AutoTokenizer.from_pretrained(model_name)
    model = AutoModel.from_pretrained(model_name, 
                                      output_attentions=True, 
                                      attn_implementation="eager")
    model_pairs.append((name,tokenizer,model,simple_names[m]))
    m+=1
    

print("Models list:")
count = 0
for model in model_pairs:
    print(f"n.{count} {model[0]}")
    count+=1

Models list:
n.0 bert-base-uncased
n.1 Musixmatch/umberto-commoncrawl-cased-v1
n.2 Musixmatch/umberto-wikipedia-uncased-v1
n.3 idb-ita/gilberto-uncased-from-camembert
n.4 m-polignano-uniba/bert_uncased_L-12_H-768_A-12_italian_alb3rt0

"""
the lines in this code block are to be run separately:
    - first line saves the current dataframe to kaggle's working directory, from where it can be downloaded and added to the notebook's datasets 
    - second line loads a dataframe from one of the uploaded datasets: this is necessary so that the preprocessed dataframe is quickly accessible
        e.g., in coref_df_tf: i) all pre-processing has been done, ii) cosine similarity for the 4 models has been processed for the 3 different scenarios
"""

# save dataframe for future quick access
coref_df_tf.to_csv("/kaggle/working/coref_df_tf_processed+cosine-sim_Ucc-Uw-G-A.csv", index=False)

# quick access of dataframe: download from imported dataframes
# this dataframe does not need to be worked at all, all processing has been carried out: start from create split cell
import pandas as pd
df = pd.read_csv("/kaggle/input/lmc-final-processed-with-cosine-similarity/coref_df_tf_processedcosine-sim_Ucc-Uw-G-A.csv")
df.head(3)

coref_df_tf = df.copy()
coref_df = coref_df_tf[coref_df_tf['label']==1]
#coref_df_tf = coref_df_tf.drop('column_name', axis=1) # in case a column needs to be dropped


# count how many NaN values there are per column in the dataframe
#print(coref_df_tf.isna().sum())
# count how many antecedent and mention are the same
#print((coref_df_tf['antecedent'] == coref_df_tf['mention']).sum())

## import dataset, assign column names, create sentence_id column
## sentence_id is needed to group sentences together which will be the examples of the dataframe later

import pandas as pd

# filepath = "/kaggle/input/document-wp-0105/document-wp_0105.txt" # small sample (1 document)
filepath = "/kaggle/input/livememorycorpus/it-train.v3amin.txt"

column_names = [
    'token_id',             # id for each token in sentence
    'word',                 # token surface form
    'ph1',                  # (empty)
    'lemma',                # token lemma form
    'ph2',                  # (empty)
    'pos',                  # POS / syntactic class
    'ph3',                  # (empty)
    'morpho_features',      # morphological features
    'ph4',                  # (empty)
    'dep_head_id',          # dependency head index
    'ph5',                  # (empty)
    'dep_rel',              # dependency relation
    'onto_type',            # ontological type
    'ph6',                  # (empty)
    'ph7',                  # (empty)
    'ph8',                  # (empty)
    'coref_col1',           # main coreference annotation
    'coref_col2',           # alternate annotation or adjudicated
    'coref_chain_info'      # coreference chain indicator
]

df = pd.read_csv(
    filepath,
    sep="\t",
    names=column_names,
    comment="#", # skips lines starting with #
    engine="python",
    encoding="latin1",
    quoting=3, # disable quote parsing
    dtype=str # treat everything as string
)

df['token_id'] = pd.to_numeric(df['token_id'], errors="coerce") # convert token column to int
#df['Dep_Head_ID'] = pd.to_numeric(df['Dep_Head_ID'], errors="coerce")

# create a sentence_id column to group the sentences
# token_id==1 marks the start of a new sentence 
df['sentence_id'] = (df['token_id'] == 1).cumsum() - 1 
# reorder columns
cols = df.columns.tolist()
cols.insert(1, cols.pop(cols.index('sentence_id')))
df = df[cols]

df.head(10)

# clean dataframe (drop empty columns)

drop_columns = ['ph1', 'ph2', 'ph3', 'ph4', 'ph5', 'ph6', 'ph7', 'ph8', ]

for c in drop_columns: 
    df = df.drop(c, axis=1)

## process dataframe
    # add true examples: tracking of coreference chains in same sentence;
    # add false examples: random sampling from the sentence's other chain's spans; 
    # prioritise V mentions in false examples.
    # oversample the false examples, later pruning is carried out
## output is a dataframe that contains i) the sentence, ii) the antecedent and mention contained, 
#                                    iii) whether the mention is a true or false mention, and iv) whether the mention is from pro-drop

import pandas as pd
import string
import re
import random

RNG = random.Random(42)

def rebuild_sentence(tokens):
    """
    Rebuilds a sentence with the right formatting (using the right spacing before and after punctuation).
    """
    sentence = ""
    for i, token in enumerate(tokens):
        if token is None:
            token = ""

        # no space before closing punctuation or sentence-final punctuation
        if token in [".", ",", ";", ":", "!", "?", ")", "»", "]"]:
            sentence += token

        # no space after opening punctuation
        elif i > 0 and tokens[i - 1] in ["(", "«", "["]:
            sentence += token

        # no space around apostrophes
        elif token.startswith("'") or (i > 0 and tokens[i - 1].endswith("'")):
            sentence += token

        else:
            if sentence:
                sentence += " "
            sentence += token
    return sentence


def process_coref_dataframe(df):
    """
    Goes through the dataframe and extracts:
        the sentence, 
        the antecedent and the mention (processed with info from 'coref_chain_info'),
        whether the mention is instance of pro-drop (whether POS of word of interest is =='V')
    """
    output = []
    current_sentence = []
    current_corefs = {}
    sentence_tokens = []

    def get_spans(mentions):
        starts, ends = [], []
        for m in mentions:
            if isinstance(m, tuple):
                if m[0] == 'start': starts.append(m[1])
                if m[0] == 'end': ends.append(m[1])
            else:
                starts.append(m)
                ends.append(m)
        return list(zip(starts, ends))
    
        
    for i, row in df.iterrows():
        token_id = int(row['token_id'])
        word = row['word']
        pos = row['pos']
        coref = row['coref_chain_info']

        # handle sentence splitting
        if token_id == 1 and sentence_tokens:
            sentence_text = rebuild_sentence(sentence_tokens)

            # build spans for all coreference chains
            all_spans = {cid: get_spans(mentions) for cid, mentions in current_corefs.items()}

            # process each chain (positives + negatives)
            for chain_id, spans in all_spans.items():
                if len(spans) >= 2:
                    antecedent_span = spans[0]
                    
                    for mention_span in spans[1:]:
                        # antecedent tokens (same for both examples)
                        ant_tokens = [t if t is not None else "" for t in sentence_tokens[antecedent_span[0]:antecedent_span[1]+1]]
                    
                        # TRUE mention tokens and POS check
                        men_tokens = [t if t is not None else "" for t in sentence_tokens[mention_span[0]:mention_span[1]+1]]
                        mention_idx = i - len(sentence_tokens) + mention_span[0]
                        mention_pos = df.iloc[mention_idx]['pos']
                        is_dropped = mention_pos.startswith("V")
                    
                        # TRUE example
                        output.append({
                            'text': sentence_text,
                            'antecedent': rebuild_sentence(ant_tokens),
                            'mention': rebuild_sentence(men_tokens),
                            'dropped_subject': is_dropped,
                            'label': 1
                        })
                    
                        # FALSE example — same antecedent, but different mention (from a different coref chain)
                        # build a list of all false mention spans (other chains)
                        false_candidates = []
                        for other_chain_id, other_spans in all_spans.items():
                            if other_chain_id == chain_id or not other_spans:
                                continue
                            false_candidates.extend(other_spans)
                        
                        # remove duplicate spans 
                        false_candidates = list({(s, e): None for (s, e) in false_candidates}.keys())
                        
                        # sample 2 random false mentions without replacement
                        num_neg = min(2, len(false_candidates))
                        
                        # prioritize dropped subject candidates (POS starts with "V")
                        verb_candidates = []
                        non_verb_candidates = []
                        for s, e in false_candidates:
                            idx = i - len(sentence_tokens) + s
                            pos_tag = df.iloc[idx]['pos']
                            (verb_candidates if pos_tag.startswith("V") else non_verb_candidates).append((s, e))
                        
                        # sample more from verb_candidates if available
                        n_verb = min(num_neg, len(verb_candidates))
                        n_nonverb = num_neg - n_verb
                        
                        sampled_false = []
                        if n_verb > 0:
                            sampled_false += RNG.sample(verb_candidates, k=n_verb)
                        if n_nonverb > 0 and len(non_verb_candidates) >= n_nonverb:
                            sampled_false += RNG.sample(non_verb_candidates, k=n_nonverb)


                        for false_mention_span in sampled_false:
                            false_mention_tokens = [
                                t if t is not None else ""
                                for t in sentence_tokens[false_mention_span[0]:false_mention_span[1]+1]
                            ]
                            false_mention_idx = i - len(sentence_tokens) + false_mention_span[0]
                            false_mention_pos = df.iloc[false_mention_idx]['pos']
                            is_dropped_false = false_mention_pos.startswith("V")
                        
                            output.append({
                                'text': sentence_text,
                                'antecedent': rebuild_sentence(ant_tokens),
                                'mention': rebuild_sentence(false_mention_tokens),
                                'dropped_subject': is_dropped_false,
                                'label': 0
                            })


            
            # reset for new sentence
            sentence_tokens = []
            current_corefs = {}

        sentence_tokens.append(word)
        
        # coreference parsing
        if pd.notna(coref) and coref != "_":
            for item in coref.split("|"):
                item = item.strip()
                if item.startswith('(') and item.endswith(')'):
                    cid = item[1:-1]
                    current_corefs.setdefault(cid, []).append(len(sentence_tokens) - 1)
                elif item.startswith('('):
                    cid = item[1:]
                    current_corefs.setdefault(cid, []).append(('start', len(sentence_tokens) - 1))
                elif item.endswith(')'):
                    cid = item[:-1]
                    current_corefs.setdefault(cid, []).append(('end', len(sentence_tokens) - 1))

    # handle last sentence
    if sentence_tokens:
        sentence_text = rebuild_sentence(sentence_tokens)
        all_spans = {cid: get_spans(mentions) for cid, mentions in current_corefs.items()}

        for chain_id, spans in all_spans.items():
            if len(spans) >= 2:
                antecedent_span = spans[0]
                
                # true mention
                for mention_span in spans[1:]:
                    ant_tokens = sentence_tokens[antecedent_span[0]:antecedent_span[1]+1]
                    men_tokens = sentence_tokens[mention_span[0]:mention_span[1]+1]
                    mention_pos = df.iloc[i - len(sentence_tokens) + mention_span[0]]['pos']
                    dropped = mention_pos.startswith("V")
                    output.append({
                        'text': sentence_text,
                        'antecedent': rebuild_sentence(ant_tokens),
                        'mention': rebuild_sentence(men_tokens),
                        'dropped_subject': dropped,
                        'label': 1
                    })

                    # false mention
                    false_candidates = []
                    for other_chain_id, other_spans in all_spans.items():
                        if other_chain_id == chain_id or not other_spans:
                            continue
                        false_candidates.extend(other_spans)
                    
                    false_candidates = list({(s, e): None for (s, e) in false_candidates}.keys())
                    
                    num_neg = min(2, len(false_candidates))
                    sampled_false = RNG.sample(false_candidates, k=num_neg) if num_neg > 0 else []
                    
                    for false_mention_span in sampled_false:
                        false_mention_tokens = [
                            t if t is not None else ""
                            for t in sentence_tokens[false_mention_span[0]:false_mention_span[1]+1]
                        ]
                        false_mention_idx = i - len(sentence_tokens) + false_mention_span[0]
                        false_mention_pos = df.iloc[false_mention_idx]['pos']
                        is_dropped_false = false_mention_pos.startswith("V")
                    
                        output.append({
                            'text': sentence_text,
                            'antecedent': rebuild_sentence(ant_tokens),
                            'mention': rebuild_sentence(false_mention_tokens),
                            'dropped_subject': is_dropped_false,
                            'label': 0
                        })

    
    return pd.DataFrame(output)


coref_df_tf = process_coref_dataframe(df)
print(coref_df_tf['label'].value_counts())
coref_df_tf.head(5)

label
0    5203
1    2609
Name: count, dtype: int64

## visualisation of balance of non-dropped and dropped mentions between true and false labels (oversampling in false labels) 

coref_df = coref_df_tf[coref_df_tf['label'] == 1].copy()
coref_df_f = coref_df_tf[coref_df_tf['label'] == 0].copy()
print(f"Distribution in true labels:\n{coref_df['dropped_subject'].value_counts()}\n")
print(f"Distribution in false labels:\n{coref_df_f['dropped_subject'].value_counts()}")

Distribution in true labels:
dropped_subject
False    1988
True      621
Name: count, dtype: int64

Distribution in false labels:
dropped_subject
False    4435
True      768
Name: count, dtype: int64

## add positional information for antecedent and mention 

import pandas as pd
import re

def get_all_offsets(text, substring):
    """
    Checks all occurrences of target in text. 
    """
    return [m.start() for m in re.finditer(re.escape(substring), text)]

def expand_with_offsets(df):
    """
    Goes through the dataset and gets information regarding offset of antecedent and mention (all occurrences in text).
    """
    expanded_rows = []
    for _, row in df.iterrows():
        sentence = row['text']
        antecedent = row['antecedent']
        mention = row['mention']
        label = row['label']
        dropped = row.get('dropped_subject', None)

        ant_offsets = get_all_offsets(sentence, antecedent)
        men_offsets = get_all_offsets(sentence, mention)

        if not ant_offsets:
            print(f"[Row {idx}] Antecedent '{antecedent}' not found in sentence:\n{sentence}\n")
        if not men_offsets:
            print(f"[Row {idx}] Mention '{mention}' not found in sentence:\n{sentence}\n")

        for ant_offset in ant_offsets:
            for men_offset in men_offsets:
                expanded_rows.append({
                    'text': sentence,
                    'antecedent': antecedent,
                    'mention': mention,
                    'label': label,
                    'antecedent_offset': ant_offset,
                    'mention_offset': men_offset,
                    'dropped_subject': dropped
                })

    return pd.DataFrame(expanded_rows)

coref_df = expand_with_offsets(coref_df)
#coref_df.head(5)

coref_df_tf = expand_with_offsets(coref_df_tf)
print(coref_df_tf['label'].value_counts())
coref_df_tf.head(5)

label
0    6152
1    3761
Name: count, dtype: int64

## add string label 

coref_df_tf['label_str'] = coref_df_tf['label'].map({
    1: 'True-mention',
    0: 'False-mention'
})

coref_df_tf['dropped_subject_str'] = coref_df_tf['dropped_subject'].map({
    True: 'Dropped',
    False: 'Non-Dropped'
})


## reorder columns
cols = coref_df_tf.columns.tolist()
cols.insert(7, cols.pop(cols.index('dropped_subject_str')))
cols.insert(1, cols.pop(cols.index('antecedent_offset')))
cols.insert(3, cols.pop(cols.index('mention_offset')))
cols.insert(6, cols.pop(cols.index('label_str')))
coref_df_tf = coref_df_tf[cols]
coref_df_tf.head(1)

## graph visualisation of length of sentences and relative position of antecedent and mention throughout the dataframe, divided in true mentions and false mentions

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from matplotlib.lines import Line2D

df = coref_df_tf.copy()
df["text_len"] = df["text"].apply(len)

# sample to keep plot readable
max_examples = len(df)//5
sampled = df.sample(n=min(len(df), max_examples), random_state=42).reset_index(drop=True)

fig, axes = plt.subplots(1, 2, figsize=(14, 8), sharex=True, sharey=True, constrained_layout=True)
label_values = [1, 0]
label_titles = {1: "label = 1", 0: "label = 0"}
marker_colors = {"antecedent": "tab:red", "mention": "tab:green"}

for ax, lbl in zip(axes, label_values):
    subset = sampled[sampled["label"] == lbl].reset_index(drop=True)
    for i, (_, row) in enumerate(subset.iterrows()):
        length = row["text_len"]
        a_off = row["antecedent_offset"]
        m_off = row["mention_offset"]

        # baseline sentence length
        ax.hlines(y=i, xmin=0, xmax=length, color="lightgray", linewidth=2)

        # antecedent marker (vertical tick)
        if pd.notna(a_off):
            ax.plot(a_off, i, marker=".", markersize=6, color=marker_colors["antecedent"])

        # mention marker (underscore)
        if pd.notna(m_off):
            ax.plot(m_off, i, marker=".", markersize=6, color=marker_colors["mention"])

    ax.set_title(f"{label_titles[lbl]} (n={len(subset)})")
    ax.set_xlabel("Token index")
    ax.set_ylabel("Example index")
    ax.set_ylim(-1, max(len(subset), 1))  # ensure visible

# shared legend
legend_elements = [
    Line2D([0], [0], color="lightgray", lw=2, label="Sentence length"),
    Line2D([0], [0], marker=".", color=marker_colors["antecedent"], linestyle="None", markersize=14, label="Antecedent"),
    Line2D([0], [0], marker=".", color=marker_colors["mention"], linestyle="None", markersize=14, label="Mention"),
]
axes[-1].legend(handles=legend_elements, loc="upper right")

fig.suptitle("Antecedent / Mention Offsets by Label (sampled)", fontsize=16, y=1.02)
plt.show()

## check whether there is imbalance given by antecedet-mention order  

print("Balance of position of antecedent and mention (before)")
true_df = coref_df_tf[coref_df_tf['label'] == True]
false_df = coref_df_tf[coref_df_tf['label'] == False]

am = (true_df['antecedent_offset'] > true_df['mention_offset']).sum()
ma = (true_df['antecedent_offset'] < true_df['mention_offset']).sum()
print(f"True: ant>men: {am} -- ant<ment: {ma}")

am = (false_df['antecedent_offset'] > false_df['mention_offset']).sum()
ma = (false_df['antecedent_offset'] < false_df['mention_offset']).sum()
print(f"False: ant>men: {am} -- ant<ment: {ma}")

# loop over dataframe and swap antecedent and mention labels if antecedent_offset>mention_offset
for i,row in coref_df_tf.iterrows():
    if row['antecedent_offset'] > row['mention_offset']:
        mention = row['mention']
        mention_offset = row['mention_offset']
        antecedent = row['antecedent']
        antecedent_offset = row['antecedent_offset']
        coref_df_tf.at[i, 'mention'] = antecedent
        coref_df_tf.at[i, 'mention_offset'] = antecedent_offset
        coref_df_tf.at[i, 'antecedent'] = mention
        coref_df_tf.at[i, 'antecedent_offset'] = mention_offset


print("\nBalance of position of antecedent and mention (after)")
true_df = coref_df_tf[coref_df_tf['label'] == True]
false_df = coref_df_tf[coref_df_tf['label'] == False]

am = (true_df['antecedent_offset'] > true_df['mention_offset']).sum()
ma = (true_df['antecedent_offset'] < true_df['mention_offset']).sum()
print(f"True: ant>men: {am} -- ant<ment: {ma}")

am = (false_df['antecedent_offset'] > false_df['mention_offset']).sum()
ma = (false_df['antecedent_offset'] < false_df['mention_offset']).sum()
print(f"False: ant>men: {am} -- ant<ment: {ma}")

Balance of position of antecedent and mention (before)
True: ant>men: 346 -- ant<ment: 2380
False: ant>men: 1223 -- ant<ment: 2597

Balance of position of antecedent and mention (after)
True: ant>men: 0 -- ant<ment: 2726
False: ant>men: 0 -- ant<ment: 3820

## visualisation of problematic examples

examples = [5,8,13,16]

for n in examples:
    for i, row in coref_df[n:n+1].iterrows():
        print(f"index: {i}.")
        print(f"sentence: {row['text']}")
        print(f"antecedent: {row['antecedent']}")
        print(f"mention: {row['mention']}")
        print(f"pro-drop: {row['dropped_subject']}")
        print("----------------")

index: 15.
sentence: Il castello, passato come feudo alla famiglia Wolkenstein, allo scoppio della Prima Guerra Mondiale venne abbandonato dalla stessa famiglia, e venne occupato dal Comando delle truppe italiane in Valsugana.
antecedent: Il castello, passato come feudo alla famiglia Wolkenstein
mention: venne
pro-drop: True
----------------
index: 18.
sentence: Il castello, passato come feudo alla famiglia Wolkenstein, allo scoppio della Prima Guerra Mondiale venne abbandonato dalla stessa famiglia, e venne occupato dal Comando delle truppe italiane in Valsugana.
antecedent: Il castello, passato come feudo alla famiglia Wolkenstein
mention: venne
pro-drop: True
----------------
index: 34.
sentence: L'ingresso del complesso, quello più antico, era situato verso il paese e vi si poteva accedere grazie ad una torre di guardia, ancora oggi esistente, su cui vi risiedono gli stemmi dei vari casati: i Trapp, i Capitani Tirolesi, i Conti del Tirolo, gli Scaligeri, i Da Carrara e i Wolkenstein.
antecedent: del complesso
mention: vi
pro-drop: False
----------------
index: 39.
sentence: L'ingresso del complesso, quello più antico, era situato verso il paese e vi si poteva accedere grazie ad una torre di guardia, ancora oggi esistente, su cui vi risiedono gli stemmi dei vari casati: i Trapp, i Capitani Tirolesi, i Conti del Tirolo, gli Scaligeri, i Da Carrara e i Wolkenstein.
antecedent: una torre di guardia, ancora oggi esistente, su cui
mention: cui vi
pro-drop: False
----------------

## drop too long mentions + keep only strings before comma in antecedents and mentions

def clean_examples(df,title):
    """
    Iterates through the dataframe and:
        if mention text has a comma, it keeps only the text before the comma
        if mention text is longer than 3 words, it delets the whole example 
        if antecedent text has a comma, it keeps only the text before the comma
    """
    lenght_pre = len(df)
    to_drop = []

    for i,row in df.iterrows():
        # keep only text before comma in mention text
        if "," in row['mention']:
            new_text = row['mention'].split(",")[0]
            df.at[i, 'mention'] = new_text

        # keep only mentions shorter or equal than 3 words
        if len(row['mention'].split()) > 3:
            to_drop.append(i)

        # keep only text before comma in antecedent text
        elif "," in row['antecedent']:
            new_text = row['antecedent'].split(",")[0]
            df.at[i, 'antecedent'] = new_text

    df = df.drop(to_drop).reset_index(drop=True)
    print(f"Examples dropped {title}: {lenght_pre-len(df)}")
    return(df)

coref_df = clean_examples(coref_df,"'coref-df only true'")
coref_df_tf = clean_examples(coref_df_tf,"'coref-df true+false'")

Examples dropped 'coref-df only true': 688
Examples dropped 'coref-df true+false': 3014

## count how many times cosine similarity is 1 in the dataset (probably same occurrence between the two words)

# ! to run this first partthe dataset with cosine similarity must be loaded first !
#column = 'similarity-avg-AlBERTo'
#count_ones = ((coref_df_tf[column] < 1.000001) & (coref_df_tf[column] > 0.999999)).sum()
#print(f"Number of cosine similarity equal to 1 (0.999999>sim>1.000001) in '{column}': {count_ones}")


## check how many same occurrences there are between antecedent and mention
same_occurrences = (coref_df_tf['antecedent'].str.strip().str.lower() 
                    == coref_df_tf['mention'].str.strip().str.lower()).sum()
print(f"Number of same occurrences between antecedent and mention: {same_occurrences}")


## check distribution of the same occurrences between true and false antecedent labels
mask = (
    coref_df_tf['antecedent'].str.strip().str.lower()
    == coref_df_tf['mention'].str.strip().str.lower()
)
new_df = coref_df_tf.loc[mask].copy().reset_index(drop=True)

print("\nDistribution of same occurrences between true and false antecedent labels:")
print(new_df["label_str"].value_counts())


## check how many times the mention is contained in the antecedent
in_occurrences = coref_df_tf.apply(
    lambda r: (
        isinstance(r['mention'], str)
        and isinstance(r['antecedent'], str)
        and r['mention'].strip().lower() in r['antecedent'].strip().lower()
    ),
    axis=1
).sum()
print(f"\nNumber of occurrences of mention in antecedent: {in_occurrences}")

Number of same occurrences between antecedent and mention: 749

Distribution of same occurrences between true and false antecedent labels:
label_str
True-mention    749
Name: count, dtype: int64

Number of occurrences of mention in antecedent: 1121

## clean dataframe of mentions that are contained in the antecedent (and thus of same mention-antecedent pairs)
## this will reduce the cases of cosine_similarity==1

mask = coref_df_tf.apply(
    lambda r: isinstance(r['mention'], str)
              and isinstance(r['antecedent'], str)
              and r['mention'].strip().lower() in r['antecedent'].strip().lower(),
    axis=1
)

coref_df_tf = coref_df_tf.loc[~mask].reset_index(drop=True)

## balance of true and false labels in new df

print("Balance of true and false labels in new dataframe:")
coref_df_tf["label_str"].value_counts()

Balance of true and false labels in new dataframe:

label_str
False-mention    3649
True-mention     2129
Name: count, dtype: int64

coref_df = coref_df_tf[coref_df_tf['label']==1].copy()
coref_df.reset_index(drop=True, inplace=True)

## balance out dataset before splitting

dropped_true = ((coref_df_tf['dropped_subject'] == True) & (coref_df_tf['label'] == 1)).sum()
dropped_false = ((coref_df_tf['dropped_subject'] == True) & (coref_df_tf['label'] == 0)).sum() 
dropped_subject_diff = abs(dropped_true - dropped_false)

true_l = (coref_df_tf['label'] == True).sum()
false_l = (coref_df_tf['label'] == False).sum()
true_false_diff = max(0, abs(true_l - false_l) - dropped_subject_diff)


candidates_drop = coref_df_tf[(coref_df_tf['dropped_subject'] == True) & (coref_df_tf['label'] == 0)] # drop dropped mentions in false labels 
candidates_main = coref_df_tf[(coref_df_tf['label'] == 0) & (coref_df_tf['dropped_subject'] != True)] # drop false lables (no drop mentions!)


# remove in dropped
n_remove_drop = min(dropped_subject_diff, len(candidates_drop)) # if fewer than n examples
to_remove_drop = candidates_drop.sample(n=n_remove_drop, random_state=42)
# remove in main
n_remove_main = min(true_false_diff, len(candidates_main))
to_remove_main = candidates_main.sample(n=n_remove_main, random_state=42)

coref_df_tf = coref_df_tf.drop(to_remove_drop.index).drop(to_remove_main.index).reset_index(drop=True)

print(f"Dropped examples:")
print(f"{dropped_subject_diff}: false dropped mention")
print(f"{true_false_diff}: false label")

Dropped examples:
143: false dropped mention
1377: false label

## work with dataframe split for ease of computing 
#split_coref_df = coref_df[:1000]

## create different splits from the original dataframe depending on different variables in order to later use them for visualisations

## divide original dataframe (only true labels) in two dataframes containing i) mentions with no pro-drop; and ii) mentions with pro-drop

df_no_dropped = coref_df[coref_df['dropped_subject'] == False] # mention was not POS tagged with 'V'
df_dropped = coref_df[coref_df['dropped_subject'] == True] # mention was POS tagged with 'V'

print(f"dataframe:\tlenght:\n"
      f"coref_df (true only)\n"
      f"-----------------------\n"
        f"df no-pro-drop\t{len(df_no_dropped)}\n"
        f"df pro-drop\t{len(df_dropped)}\n\n")


## divide original dataframe (true and false labels) in two dataframes containing i) mentions with no pro-drop; and ii) mentions with pro-drop

df_tf_no_dropped = coref_df_tf[coref_df_tf['dropped_subject'] == False] # mention was not POS tagged with 'V'
df_tf_dropped = coref_df_tf[coref_df_tf['dropped_subject'] == True] # mention was POS tagged with 'V'

print(f"dataframe:\tlenght:\n"
      f"coref_df_tf (true+false)\n"
      f"-----------------------\n"
        f"df no-pro-drop\t{len(df_tf_no_dropped)}\n"
        f"df pro-drop\t{len(df_tf_dropped)}\n\n")


## divide original dataframe (true and false labels) in two dataframes containing i) true mentions; and ii) false mentions

df_tf_true = coref_df_tf[coref_df_tf['label'] == True] # label==1 : correct mention
df_tf_false = coref_df_tf[coref_df_tf['label'] == False] # label==0 : false mention

print(f"dataframe:\tlenght:\n"
      f"coref_df_tf (true+false)\n"
      f"-----------------------\n"
        f"df true label\t{len(df_tf_true)}\n"
        f"df false label\t{len(df_tf_false)}\n\n")


## create two sub-dataframes which contain i) mentions with pro-drop in true label examples, and ii) mentions with pro-drop in false label examples
df_tf_true_drop = df_tf_true[df_tf_true['dropped_subject'] == True] # mention was POS tagged with 'V'
df_tf_false_drop = df_tf_false[df_tf_false['dropped_subject'] == True] # mention was POS tagged with 'V'

print(f"dataframe:\tlenght:\n"
      f"df_tf_true/false\n"
      f"-----------------------\n"
        f"df-true drop\t{len(df_tf_true_drop)}\n"
        f"df-false drop\t{len(df_tf_false_drop)}\n\n")

dataframe:	lenght:
coref_df (true only)
-----------------------
df no-pro-drop	1477
df pro-drop	652


dataframe:	lenght:
coref_df_tf (true+false)
-----------------------
df no-pro-drop	2954
df pro-drop	1304


dataframe:	lenght:
coref_df_tf (true+false)
-----------------------
df true label	2129
df false label	2129


dataframe:	lenght:
df_tf_true/false
-----------------------
df-true drop	652
df-false drop	652

## visualisation: iteration through WHOLE DATASET
# model: iteration (4 Italian models)
# dataset: coref_df (only true labels) 

import torch
import matplotlib.pyplot as plt
import seaborn as sns
from tqdm import tqdm
import numpy as np

problems = []

for m in model_pairs[1:]:

    model_name = m[0]
    print(f"model name: {model_name}")
    tokenizer = m[1]
    model = m[2]
    
    num_layers = model.config.num_hidden_layers
    num_heads = model.config.num_attention_heads
    
    # initialise accumulator
    total_attention_a2p = torch.zeros((num_layers, num_heads)) # tensor filled with 0s
    total_attention_p2a = torch.zeros((num_layers, num_heads)) 
    
    num_samples = len(coref_df)
    num_labelling_problems = 0
    num_indexing_problems = 0
    
    # iterate through dataset
    for n in tqdm(range(num_samples), desc="Processing", unit="sample", colour="green"):
        sentence = coref_df['text'][n].lower()
        antecedent = coref_df['antecedent'][n].lower()
        mention = coref_df['mention'][n].lower()
        
        # tokenise input
        inputs = tokenizer(sentence, return_tensors="pt", add_special_tokens=False)
        tokens = tokenizer.convert_ids_to_tokens(inputs["input_ids"][0])
    
        with torch.no_grad():
            outputs = model(**inputs)
            attentions = outputs.attentions  # get attention matrices
        
        # identify token indices for antecedent (needed because of subword tokenizer)
        antecedent_tokens = tokenizer.tokenize(antecedent)  # tokenise antecedent itself
        antecedent_indices = []
        for i in range(len(tokens)):
            if tokens[i : i + len(antecedent_tokens)] == antecedent_tokens:
                antecedent_indices = list(range(i, i + len(antecedent_tokens)))
                break  # stop after first match
        # identify token indices for mention
        mention_tokens = tokenizer.tokenize(mention)  # tokenise mention itself
        mention_indices = []
        for i in range(len(tokens)):
            if tokens[i : i + len(mention_tokens)] == mention_tokens:
                mention_indices = list(range(i, i + len(mention_tokens))) # correct to plural (in whole code)
                break  # stop after first match
        
        # check if antecedent was found
        if not antecedent_indices:
            #print(f"Skipping example: antecedent '{antecedent}' not found in tokenized text.")
            #print("Sentence: ",coref_df['text'][n])
            #print(f"Example number: {n}.\n")
            num_labelling_problems+=1
            # creating a dictionary instead of printing problems for a clearer output view
            problems.append({
                'model' : model_name,
                'error' : "antecedent not found",
                'num' : n,
                'example' : coref_df['text'][n],
                'antecedent' : antecedent
            })
            continue  # avoid adding NaN values (results in Error)

        # check if antecedent was found
        if not mention_indices:
            #print(f"Skipping example: mention '{mention}' not found in tokenized text.")
            #print("Sentence: ",coref_df['text'][n])
            #print(f"Example number: {n}.\n")
            num_labelling_problems+=1
            # creating a dictionary instead of printing problems for a clearer output view
            problems.append({
                'model' : model_name,
                'error' : "mention not found",
                'num' : n,
                'example' : coref_df['text'][n],
                'mention' : mention
            })
            continue  # avoid adding NaN values (results in Error)

        # check if mention indices are within tokenised sentence bounds
        if any(idx >= len(tokens) for idx in mention_indices):
            #print(f"Out of bounds mention_indices at example {n}:")
            #print(f"Sentence: {row['text']}")
            #print(f"Mention: {row['mention']}")
            #print(f"Mention index: {mention_indices}, Sentence length: {len(tokens)}")
            num_indexing_problems+=1
            problems.append({
                'model' : model_name,
                'error' : "out of bounds mention index",
                'num' : n,
                'example' : coref_df['text'][n],
                'mention' : mention,
                'mention-idx, sentence-len' : (mention_indices,len(tokens))
            })
            continue
            

        # accumulate attention scores
        for layer in range(num_layers):
            for head in range(num_heads):
                attn_matrix = attentions[layer][0, head] # attention matrix of shape [seq_len, seq_len]
                attention_value_a2p = attn_matrix[np.ix_(antecedent_indices, mention_indices)].mean()
                attention_value_p2a = attn_matrix[np.ix_(mention_indices, antecedent_indices)].mean()
                total_attention_a2p[layer, head] += attention_value_a2p
                total_attention_p2a[layer, head] += attention_value_p2a
    
    # compute average attention across dataset
    valid_samples = num_samples - num_labelling_problems - num_indexing_problems
    avg_attention_a2p = total_attention_a2p / valid_samples
    avg_attention_p2a = total_attention_p2a / valid_samples
    
    # convert to NumPy for visualisation
    avg_attention_a2p = avg_attention_a2p.numpy()
    avg_attention_p2a = avg_attention_p2a.numpy()
    
    print(f"Number of labelling problems found: {num_labelling_problems}")
    print(f"Number of indexing problems found: {num_indexing_problems}")
    
    
    ## visualisation 

    vmin = min(avg_attention_a2p.min(), avg_attention_p2a.min())
    vmax = max(avg_attention_a2p.max(), avg_attention_p2a.max())

    # plot average attention heatmap
    fig, axs = plt.subplots(1, 2, figsize=(16, 6))  # 1 row, 2 columns

    # overall title
    fig.suptitle("Attention Patterns: True-lables-only Dataset", fontsize=16, y=0.98)
    
    # heatmap: antecedent > mention
    sns.heatmap(avg_attention_a2p, annot=True, fmt=".2f", cmap="viridis", # "coolwarm"
                xticklabels=[f"H{i}" for i in range(num_heads)],
                yticklabels=[f"L{i}" for i in range(num_layers)],
                ax=axs[0], vmin=vmin, vmax=vmax)
    axs[0].set_title("Average Attention from Antecedent → Mention")
    axs[0].set_xlabel("Heads")
    axs[0].set_ylabel("Layers")
    
    # heatmap: mention > antecedent
    sns.heatmap(avg_attention_p2a, annot=True, fmt=".2f", cmap="viridis", # "coolwarm"
                xticklabels=[f"H{i}" for i in range(num_heads)],
                yticklabels=[f"L{i}" for i in range(num_layers)],
                ax=axs[1], vmin=vmin, vmax=vmax)
    axs[1].set_title("Average Attention from Mention → Antecedent")
    axs[1].set_xlabel("Heads")
    axs[1].set_ylabel("Layers")
    
    plt.tight_layout()
    plt.show()

model name: Musixmatch/umberto-commoncrawl-cased-v1

Processing: 100%|██████████| 2129/2129 [03:59<00:00,  8.89sample/s]

Number of labelling problems found: 104
Number of indexing problems found: 0

model name: Musixmatch/umberto-wikipedia-uncased-v1

Processing: 100%|██████████| 2129/2129 [03:57<00:00,  8.97sample/s]

Number of labelling problems found: 59
Number of indexing problems found: 0

model name: idb-ita/gilberto-uncased-from-camembert

Processing: 100%|██████████| 2129/2129 [03:53<00:00,  9.11sample/s]

Number of labelling problems found: 104
Number of indexing problems found: 0

model name: m-polignano-uniba/bert_uncased_L-12_H-768_A-12_italian_alb3rt0

## visualisation of 3 random problem examples from each model

import pandas as pd

problems_df = pd.DataFrame(problems)

samples = []
# group by model and sample (at least) 3
for model, group in problems_df.groupby("model"):
    sampled = group.sample(n=min(3, len(group)), random_state=43)
    samples.append(sampled)
samples = pd.concat(samples, axis=0).reset_index(drop=True)

# print sampled
for _,row in samples.iterrows():
    print(f"model: {row['model']}")
    print(f"error: {row['error']}")
    print(f"example: {row['num']}. {row['example']}")
    val = row['antecedent']
    if pd.isna(val) or val == "":
        val = row['mention']
    print(f"problem: {val}")
    print("---------------------")

model: Musixmatch/umberto-commoncrawl-cased-v1
error: antecedent not found
example: 36. Papi coinvolti nel concilio e nella sua preparazione''Leone X: Durante il suo pontificato ebbe luogo la ribellione di Lutero''Clemente VII Questi due papi di formazione rinascimentale, della famiglia dei Medici, sottovalutarono la portata della Riforma e non convocarono, come da più parti richiesto, il concilio per la riforma della chiesa.
problem: leone x
---------------------
model: Musixmatch/umberto-commoncrawl-cased-v1
error: antecedent not found
example: 219. Così, quando, nel 1838, don Pier Paolo Righler, padre spirituale del seminario teologico di Trento, fu nominato alla cattedra di professore di pastorale, il vescovo nominò a sua volta don Matteo Gottardi suo successore.
problem: nel 1838
---------------------
model: Musixmatch/umberto-commoncrawl-cased-v1
error: mention not found
example: 1567. La sede centrale dell'associazione si trova a Trento nel Palazzo Saracini - Cresseri (sec. XVI) che ospita oltre all'Organizzazione centrale, il Museo della SAT, l'Archivio storico, la Biblioteca della montagna, la Direzione provinciale del Corpo soccorso alpino S.A. T., la Sezione SAT di Trento, la S. U.S. A.T. (Sezione universitaria), il Coro della SAT, il Collegio provinciale delle Guide alpine, il Gruppo rocciatori SAT e la Scuola di alpinismo e sci alpinismo''Giorgio Graffer''.
problem: s.a. t.
---------------------
model: Musixmatch/umberto-wikipedia-uncased-v1
error: antecedent not found
example: 991. «io volevo [...] far cementare le mie statue, una dietro l'altra, davanti al municipio di Calais, proprio nel mezzo al selciato della piazza, come una corte vivente di sofferenza e sacrificio.
problem: io
---------------------
model: Musixmatch/umberto-wikipedia-uncased-v1
error: antecedent not found
example: 1642. ''Martini fu il primo a studiare la storia e la geografia Cinese con obiettivo rigore scientifico, l'ampiezza della sua conoscenza della cultura cinese, l'accuratezza delle sue ricerche, la profondità della sua comprensione dei concetti cinesi sono esempi per i moderni sinologi''.
problem: martini
---------------------
model: Musixmatch/umberto-wikipedia-uncased-v1
error: antecedent not found
example: 1641. ''Martini fu il primo a studiare la storia e la geografia Cinese con obiettivo rigore scientifico, l'ampiezza della sua conoscenza della cultura cinese, l'accuratezza delle sue ricerche, la profondità della sua comprensione dei concetti cinesi sono esempi per i moderni sinologi''.
problem: martini
---------------------
model: idb-ita/gilberto-uncased-from-camembert
error: antecedent not found
example: 36. Papi coinvolti nel concilio e nella sua preparazione''Leone X: Durante il suo pontificato ebbe luogo la ribellione di Lutero''Clemente VII Questi due papi di formazione rinascimentale, della famiglia dei Medici, sottovalutarono la portata della Riforma e non convocarono, come da più parti richiesto, il concilio per la riforma della chiesa.
problem: leone x
---------------------
model: idb-ita/gilberto-uncased-from-camembert
error: antecedent not found
example: 219. Così, quando, nel 1838, don Pier Paolo Righler, padre spirituale del seminario teologico di Trento, fu nominato alla cattedra di professore di pastorale, il vescovo nominò a sua volta don Matteo Gottardi suo successore.
problem: nel 1838
---------------------
model: idb-ita/gilberto-uncased-from-camembert
error: mention not found
example: 1567. La sede centrale dell'associazione si trova a Trento nel Palazzo Saracini - Cresseri (sec. XVI) che ospita oltre all'Organizzazione centrale, il Museo della SAT, l'Archivio storico, la Biblioteca della montagna, la Direzione provinciale del Corpo soccorso alpino S.A. T., la Sezione SAT di Trento, la S. U.S. A.T. (Sezione universitaria), il Coro della SAT, il Collegio provinciale delle Guide alpine, il Gruppo rocciatori SAT e la Scuola di alpinismo e sci alpinismo''Giorgio Graffer''.
problem: s.a. t.
---------------------

## iteration through the whole dataset after splitting it in two datasets plotted side by side for comparison
# model: iteration (4 Italian models)
# dataset: -- 

import torch
import matplotlib.pyplot as plt
import seaborn as sns
from tqdm import tqdm
import numpy as np

problems = []

# function to visualise the different datasets: args(df,label_to_plot)
def average_attention_split(df0,df1,label0,label1,overall_title):
    
    # function to compute average attention
    def compute_average_attention(df_subset):
        total_attention_a2p = torch.zeros((num_layers, num_heads)) # accumulator
        total_attention_p2a = torch.zeros((num_layers, num_heads))
        num_samples = len(df_subset)
        
        num_labelling_problems = 0
        num_indexing_problems = 0
    
        # iterate through dataset
        for n in tqdm(range(num_samples), desc="Processing", unit="sample", colour="green"):
            sentence = df_subset['text'].iloc[n].lower()
            antecedent = df_subset['antecedent'].iloc[n].lower()
            mention = df_subset['mention'].iloc[n].lower()
            
            # tokenize input
            inputs = tokenizer(sentence, return_tensors="pt", add_special_tokens=False)
            tokens = tokenizer.convert_ids_to_tokens(inputs["input_ids"][0])
    
            with torch.no_grad():
                outputs = model(**inputs)
                attentions = outputs.attentions # get attention matrices
    
            # identify token indices for antecedent (considers subword tokenizer)
            antecedent_tokens = tokenizer.tokenize(antecedent)
            antecedent_indices = []
            for i in range(len(tokens)):
                if tokens[i : i + len(antecedent_tokens)] == antecedent_tokens:
                    antecedent_indices = list(range(i, i + len(antecedent_tokens)))
                    break  
            
            # identify token index for mention
            mention_tokens = tokenizer.tokenize(mention)  # tokenise mention itself
            mention_indices = []
            for i in range(len(tokens)):
                if tokens[i : i + len(mention_tokens)] == mention_tokens:
                    mention_indices = list(range(i, i + len(mention_tokens))) # correct to plural (in whole code)
                    break  # stop after first match 
                    
            # check if antecedent was found
            if not antecedent_indices:
                #print(f"Skipping example: antecedent '{antecedent}' not found in tokenized text.")
                #print("Sentence: ", df_subset.iloc[n]['sentence'])
                #print(f"Example number: {n}.\n")
                num_labelling_problems+=1
                # creating a dictionary instead of printing problems for a clearer output view
                problems.append({
                    'model' : model_name,
                    'dataset split' : df_subset,
                    'error' : "antecedent not found",
                    'num' : n,
                    'example' : coref_df['text'][n],
                    'antecedent' : antecedent
                })
                continue  # avoid adding NaN values (results in Error)

            # check if mention was found
            if not mention_indices:
                #print(f"Skipping example: mention '{mention}' not found in tokenized text.")
                #print("Sentence: ", df_subset.iloc[n]['sentence'])
                #print(f"Example number: {n}.\n")
                num_labelling_problems+=1
                # creating a dictionary instead of printing problems for a clearer output view
                problems.append({
                    'model' : model_name,
                    'dataset split' : df_subset,
                    'error' : "mention not found",
                    'num' : n,
                    'example' : coref_df['text'][n],
                    'mention' : mention
                })
                continue  # avoid adding NaN values (results in Error)
                
            # check if mention indices are within tokenised sentence bounds
            if any(idx >= len(tokens) for idx in mention_indices):
                #print(f"Out of bounds mention_indices at example {n}:")
                #print(f"Sentence: {row['text']}")
                #print(f"Mention: {row['mention']}")
                #print(f"Mention index: {mention_indices}, Sentence length: {len(tokens)}")
                num_indexing_problems+=1
                problems.append({
                    'model' : model_name,
                    'dataset split' : df_subset,
                    'error' : "out of bounds mention index",
                    'num' : n,
                    'example' : coref_df['text'][n],
                    'mention' : mention,
                    'mention-idx, sentence-len' : (mention_indices,len(tokens))
                })
                continue

            # accumulate attention scores
            for layer in range(num_layers):
                for head in range(num_heads):
                    attn_matrix = attentions[layer][0, head] # attention matrix of shape [seq_len, seq_len]
                    attention_value_a2p = attn_matrix[np.ix_(antecedent_indices, mention_indices)].mean()
                    attention_value_p2a = attn_matrix[np.ix_(mention_indices, antecedent_indices)].mean()
                    total_attention_a2p[layer, head] += attention_value_a2p
                    total_attention_p2a[layer, head] += attention_value_p2a
                
        # compute average attention across dataset
        avg_attention_a2p = total_attention_a2p / num_samples
        avg_attention_p2a = total_attention_p2a / num_samples
        
        # convert to NumPy for visualisation
        avg_attention_a2p = avg_attention_a2p.numpy()
        avg_attention_p2a = avg_attention_p2a.numpy()

        return avg_attention_a2p, avg_attention_p2a
    
    # compute attention for both subsets and both directions
    avg_attention_0_a2p, avg_attention_0_p2a = compute_average_attention(df0)
    avg_attention_1_a2p, avg_attention_1_p2a = compute_average_attention(df1)


    ## visualisation
    
    # shared min and max for consistent color scale
    vmin = min(avg_attention_0_a2p.min(), avg_attention_0_p2a.min(), avg_attention_1_a2p.min(), avg_attention_1_p2a.min())
    vmax = max(avg_attention_0_a2p.max(), avg_attention_0_p2a.max(), avg_attention_1_a2p.max(), avg_attention_1_p2a.max())

    # plot side-by-side heatmaps
    fig, axs = plt.subplots(2, 2, figsize=(16, 12))
    
    xticks = [f"H{i}" for i in range(num_heads)]
    yticks = [f"L{i}" for i in range(num_layers)]
    
    # overall title
    fig.suptitle(overall_title, fontsize=16, y=0.98)

    # plot each heatmap
    sns.heatmap(avg_attention_0_a2p, annot=True, fmt=".2f", cmap="viridis",
                xticklabels=xticks, yticklabels=yticks, ax=axs[0, 0], vmin=vmin, vmax=vmax)
    axs[0, 0].set_title(f"Average Attention from Antecedent → Mention — {label0}")
    axs[0, 0].set_xlabel("Heads")
    axs[0, 0].set_ylabel("Layers")

    sns.heatmap(avg_attention_0_p2a, annot=True, fmt=".2f", cmap="viridis",
                xticklabels=xticks, yticklabels=yticks, ax=axs[0, 1], vmin=vmin, vmax=vmax)
    axs[0, 1].set_title(f"Average Attention from Mention → Antecedent — {label0}")
    axs[0, 1].set_xlabel("Heads")
    axs[0, 1].set_ylabel("Layers")

    sns.heatmap(avg_attention_1_a2p, annot=True, fmt=".2f", cmap="viridis",
                xticklabels=xticks, yticklabels=yticks, ax=axs[1, 0], vmin=vmin, vmax=vmax)
    axs[1, 0].set_title(f"Average Attention from Antecedent → Mention — {label1}")
    axs[1, 0].set_xlabel("Heads")
    axs[1, 0].set_ylabel("Layers")

    sns.heatmap(avg_attention_1_p2a, annot=True, fmt=".2f", cmap="viridis",
                xticklabels=xticks, yticklabels=yticks, ax=axs[1, 1], vmin=vmin, vmax=vmax)
    axs[1, 1].set_title(f"Average Attention from Mention → Antecedent — {label1}")
    axs[1, 1].set_xlabel("Heads")
    axs[1, 1].set_ylabel("Layers")

    plt.tight_layout()
    plt.show()

## attention pattern between non-dropped and dropped mentions (True-labels-only dataset)

for m in model_pairs[1:]:

    model_name = m[0]
    print(f"model name: {model_name}")
    tokenizer = m[1]
    model = m[2]

    num_layers = model.config.num_hidden_layers
    num_heads = model.config.num_attention_heads

    average_attention_split(df_no_dropped,df_dropped,"Non-dropped Mention","Dropped Mention","Attention Patterns between Non-dropped and Dropped Mentions (only True labels)")

model name: Musixmatch/umberto-commoncrawl-cased-v1

Processing: 100%|██████████| 1477/1477 [02:48<00:00,  8.76sample/s]
Processing: 100%|██████████| 652/652 [01:09<00:00,  9.35sample/s]

model name: Musixmatch/umberto-wikipedia-uncased-v1

Processing: 100%|██████████| 1477/1477 [02:49<00:00,  8.73sample/s]
Processing: 100%|██████████| 652/652 [01:09<00:00,  9.40sample/s]

model name: idb-ita/gilberto-uncased-from-camembert

Processing: 100%|██████████| 1477/1477 [02:44<00:00,  9.00sample/s]
Processing: 100%|██████████| 652/652 [01:08<00:00,  9.58sample/s]

model name: m-polignano-uniba/bert_uncased_L-12_H-768_A-12_italian_alb3rt0

Processing: 100%|██████████| 1477/1477 [02:41<00:00,  9.12sample/s]
Processing: 100%|██████████| 652/652 [01:06<00:00,  9.80sample/s]

## attention pattern between True and False mentions (True/False balanced dataset)

for m in model_pairs[1:]:

    model_name = m[0]
    print(f"model name: {model_name}")
    tokenizer = m[1]
    model = m[2]

    num_layers = model.config.num_hidden_layers
    num_heads = model.config.num_attention_heads

    average_attention_split(df_tf_true,df_tf_false,"True labels only","False labels only","Attention Patterns between True and False Mentions")

model name: Musixmatch/umberto-commoncrawl-cased-v1

Processing: 100%|██████████| 2129/2129 [04:11<00:00,  8.46sample/s]
Processing: 100%|██████████| 2129/2129 [04:19<00:00,  8.21sample/s]

model name: Musixmatch/umberto-wikipedia-uncased-v1

Processing: 100%|██████████| 2129/2129 [04:14<00:00,  8.37sample/s]
Processing: 100%|██████████| 2129/2129 [04:18<00:00,  8.23sample/s]

model name: idb-ita/gilberto-uncased-from-camembert

Processing: 100%|██████████| 2129/2129 [04:12<00:00,  8.42sample/s]
Processing: 100%|██████████| 2129/2129 [04:16<00:00,  8.29sample/s]

model name: m-polignano-uniba/bert_uncased_L-12_H-768_A-12_italian_alb3rt0

Processing: 100%|██████████| 2129/2129 [04:05<00:00,  8.66sample/s]
Processing: 100%|██████████| 2129/2129 [04:14<00:00,  8.36sample/s]

## attention pattern between True and False dropped mentions (True/False dropped mentions dataset)

for m in model_pairs[1:]:

    model_name = m[0]
    print(f"model name: {model_name}")
    tokenizer = m[1]
    model = m[2]

    num_layers = model.config.num_hidden_layers
    num_heads = model.config.num_attention_heads

    average_attention_split(df_tf_true_drop,df_tf_false_drop,"Dropped Mention (True labels)","Dropped Mention (False labels)","Attention Patterns between True Dropped Mentions and False Dropped Mentions")

model name: Musixmatch/umberto-commoncrawl-cased-v1

Processing: 100%|██████████| 652/652 [01:15<00:00,  8.65sample/s]
Processing: 100%|██████████| 652/652 [01:15<00:00,  8.69sample/s]

model name: Musixmatch/umberto-wikipedia-uncased-v1

Processing: 100%|██████████| 652/652 [01:09<00:00,  9.44sample/s]
Processing: 100%|██████████| 652/652 [01:14<00:00,  8.75sample/s]

model name: idb-ita/gilberto-uncased-from-camembert

Processing: 100%|██████████| 652/652 [01:09<00:00,  9.37sample/s]
Processing: 100%|██████████| 652/652 [01:13<00:00,  8.92sample/s]

model name: m-polignano-uniba/bert_uncased_L-12_H-768_A-12_italian_alb3rt0

Processing: 100%|██████████| 652/652 [01:08<00:00,  9.55sample/s]
Processing: 100%|██████████| 652/652 [01:13<00:00,  8.87sample/s]

## transform dataframe into dataset for integration in training pipeline

from datasets import Dataset

coref_dataset = Dataset.from_pandas(coref_df_tf) 
coref_dataset

Dataset({
    features: ['text', 'antecedent_offset', 'antecedent', 'mention_offset', 'mention', 'label', 'label_str', 'dropped_subject', 'dropped_subject_str', 'similarity-lst-UmBERToCC', 'similarity-l11-UmBERToCC', 'similarity-avg-UmBERToCC', 'similarity-lst-UmBERToW', 'similarity-l11-UmBERToW', 'similarity-avg-UmBERToW', 'similarity-lst-GilBERTo', 'similarity-l11-GilBERTo', 'similarity-avg-GilBERTo', 'similarity-lst-AlBERTo', 'similarity-l11-AlBERTo', 'similarity-avg-AlBERTo'],
    num_rows: 4258
})

## train probing classifier and visualise performance

## confusion matrix function 

from sklearn.metrics import ConfusionMatrixDisplay, confusion_matrix
import numpy as np
import matplotlib.pyplot as plt

def plot_confusion_matrix(y_preds, y_true,labels):
    cm = confusion_matrix(y_true, y_preds, normalize="true")
    fig, ax = plt.subplots(figsize=(2, 2))
    disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=["0","1"])
    disp.plot(cmap="Greens", values_format=".2f", ax=ax, colorbar=False)
    plt.title("Normalized confusion matrix")
    plt.show()


## build representations function
## feature extraction function

from transformers import AutoTokenizer, AutoModel
from tqdm import tqdm
import torch

def get_reps(example):
    text = example['text']
    men_offset = example['mention_offset']
    ant_offset = example['antecedent_offset']

    enc = tokenizer(text, return_offsets_mapping=True, return_tensors="pt")
    ids = enc['input_ids']
    offsets = enc['offset_mapping'][0]

    with torch.no_grad():
        reps = model(**{k: v for k, v in enc.items() if k!='offset_mapping'})
        hidden = reps.hidden_states[-1][0]  # last layer

    def find_rep(char_offset):
        idx = next(i for i,(s,e) in enumerate(offsets) if s <= char_offset < e)
        return hidden[idx]

    men = find_rep(men_offset)
    ant = find_rep(ant_offset)

    return {
        'men': men,
        'ant': ant,
        'lab': int(example['label'])
    }


def extract_features(dataset):
    features, labels = [], []
    for ex in tqdm(dataset, desc="Extracting features"):
        reps = get_reps(ex) 
        men = reps['men']
        ant = reps['ant']
        lab = reps['lab']
        features.append(torch.cat([men, ant]).numpy())
        labels.append(lab)

    return features, labels



## train probing classifier

from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler

summary_report = []
per_model_examples = {}  # store best/worst per model

for m in model_pairs[1:]:

    model_name = m[0]
    print(f"model name: {model_name}")
    tokenizer = m[1]
    model = AutoModel.from_pretrained(model_name, output_hidden_states=True)

    num_layers = model.config.num_hidden_layers
    num_heads = model.config.num_attention_heads

    model.eval()

    all_indices = np.arange(len(coref_dataset)) # array of original indices 
    #X_train, y_train = extract_features(train_data)
    #X_test, y_test = extract_features(test_data)
    features, labels = extract_features(coref_dataset)
    X_train, X_test, y_train, y_test, idx_train, idx_test = train_test_split( # keep indices for error analysis
        features, labels, all_indices, test_size=0.2, random_state=42 # if using only one split 
    )    
    #clf = LogisticRegression(max_iter=1000).fit(X_train, y_train)
    clf = make_pipeline(
        StandardScaler(),
        LogisticRegression(max_iter=2000, solver='lbfgs') # upped max_iter becase it was not converging
    )
    clf.fit(X_train, y_train)

    y_preds = clf.predict(X_test)

    # get confidence scores (probability of predicted class)
    if hasattr(clf, "predict_proba"):
        probs = clf.predict_proba(X_test)  # shape (n_samples, n_classes)
        # confidence == max probability per example
        confidences = np.max(probs, axis=1)
    else:
        # fallback: use decision function and map to pseudo-confidence
        decision = clf.decision_function(X_test)
        confidences = 1 / (1 + np.exp(-np.abs(decision)))  # squash to [0.5,1)

    correct_mask = (y_preds == y_test)
    incorrect_mask = ~correct_mask

    # indices
    correct_idx = np.where(correct_mask)[0]
    incorrect_idx = np.where(incorrect_mask)[0]

    # sort by confidence
    best_correct = sorted(correct_idx, key=lambda i: confidences[i], reverse=True)[:5]
    worst_incorrect = sorted(incorrect_idx, key=lambda i: confidences[i], reverse=True)[:5]
    # low-confidence correct (closest to 0.5)
    lowconf_correct = sorted(correct_idx, key=lambda i: abs(confidences[i] - 0.5))[:5]

    per_model_examples[model_name] = {
        "best_correct": best_correct,
        "worst_incorrect": worst_incorrect,
        "lowconf_correct": lowconf_correct,
        "y_test": y_test,
        "y_preds": y_preds,
        "confidences": confidences,
        "X_test": X_test,
        "idx_test": idx_test,  
    }


    # print results of probing classifier
    print(classification_report(y_test, y_preds))
    summary_report.append({
        'model': model_name,
        'report': classification_report(y_test, y_preds, output_dict=True)
    })

    # visualise performance with confusion matrix    
    # find correct and incorrect predictions
    correct_indices = np.where(y_preds == y_test)[0] # gives out an array with indices
    incorrect_indices = np.where(y_preds != y_test)[0]
    
    print(f"Number of correct predictions: {len(correct_indices)}")
    print(f"Number of incorrect predictions: {len(incorrect_indices)}")
    
    plot_confusion_matrix(y_preds, y_test, labels=[0,1])

    
    print("------------------------------------------------------------------------------------")

model name: Musixmatch/umberto-commoncrawl-cased-v1

Extracting features: 100%|██████████| 4258/4258 [07:13<00:00,  9.82it/s]

              precision    recall  f1-score   support

           0       0.71      0.73      0.72       429
           1       0.72      0.70      0.71       423

    accuracy                           0.71       852
   macro avg       0.71      0.71      0.71       852
weighted avg       0.71      0.71      0.71       852

Number of correct predictions: 609
Number of incorrect predictions: 243

------------------------------------------------------------------------------------
model name: Musixmatch/umberto-wikipedia-uncased-v1

Extracting features: 100%|██████████| 4258/4258 [08:04<00:00,  8.79it/s]

              precision    recall  f1-score   support

           0       0.68      0.72      0.70       429
           1       0.70      0.66      0.68       423

    accuracy                           0.69       852
   macro avg       0.69      0.69      0.69       852
weighted avg       0.69      0.69      0.69       852

Number of correct predictions: 589
Number of incorrect predictions: 263

------------------------------------------------------------------------------------
model name: idb-ita/gilberto-uncased-from-camembert

Extracting features: 100%|██████████| 4258/4258 [07:55<00:00,  8.95it/s]

              precision    recall  f1-score   support

           0       0.70      0.73      0.72       429
           1       0.72      0.69      0.70       423

    accuracy                           0.71       852
   macro avg       0.71      0.71      0.71       852
weighted avg       0.71      0.71      0.71       852

Number of correct predictions: 604
Number of incorrect predictions: 248

------------------------------------------------------------------------------------
model name: m-polignano-uniba/bert_uncased_L-12_H-768_A-12_italian_alb3rt0

## dummy classifier (most frequent class)

from sklearn.dummy import DummyClassifier
from sklearn.metrics import classification_report

X = X_test 
y = y_test 

dummy = DummyClassifier(strategy="most_frequent")
dummy.fit(X, y)
y_pred = dummy.predict(X)

print(classification_report(y, y_pred, zero_division=0))
summary_report.append({
        'model': "dummy classifier",
        'report': classification_report(y, y_pred, zero_division=0, output_dict=True)
    })

              precision    recall  f1-score   support

           0       0.50      1.00      0.67       429
           1       0.00      0.00      0.00       423

    accuracy                           0.50       852
   macro avg       0.25      0.50      0.33       852
weighted avg       0.25      0.50      0.34       852

## print summary of performances

print("summary of performances (f-1 scores)")
print(f"{'model':65} {'accuracy':>8}")
print("-" * 75)
for entry in summary_report:
    name = entry['model']
    acc = entry['report']['accuracy']
    print(f"{name:65} {acc:8.2f}")

summary of performances (f-1 scores)
model                                                             accuracy
---------------------------------------------------------------------------
Musixmatch/umberto-commoncrawl-cased-v1                               0.71
Musixmatch/umberto-wikipedia-uncased-v1                               0.69
idb-ita/gilberto-uncased-from-camembert                               0.71
m-polignano-uniba/bert_uncased_L-12_H-768_A-12_italian_alb3rt0        0.66
dummy classifier                                                      0.50

def build_examples_df(per_model_examples, coref_dataset):
    all_rows = []
    for model_name, ex in per_model_examples.items():
        for tag in ["best_correct", "worst_incorrect", "lowconf_correct"]:
            for i in ex[tag]:
                orig_idx = int(ex["idx_test"][i])
                row = coref_dataset[orig_idx]
                all_rows.append({
                    "model": model_name,
                    "tag": tag,
                    "text": row["text"],
                    "antecedent": row["antecedent"],
                    "mention": row["mention"],
                    "dropped_subject": row["dropped_subject"],
                    "true_label": ex["y_test"][i],
                    "pred_label": ex["y_preds"][i],
                    "confidence": ex["confidences"][i],
                })
    return pd.DataFrame(all_rows)

df_examples = build_examples_df(per_model_examples, coref_dataset)
df_examples.head(50)

current_model = ""
for i,row in df_examples.iterrows():
    m_name = row['model']
    if m_name == current_model or m_name == "":
        if m_name == "":
            print(f"Model name: {m_name}\n")
            current_model = m_name
        print(f"Sentence: {row['text']}")
        print(f"antecedent-mention: {row['antecedent']}-{row['mention']}")
        print(f"tag: {row['tag']}, true label: {row['true_label']}, pred: {row['pred_label']}, conf: {row['confidence']:.2f}")
        print("-----------------")
    else: 
        print(f"\n\nModel name: {m_name}\n")
        current_model = m_name


Model name: Musixmatch/umberto-commoncrawl-cased-v1

Sentence: Ma anche a questo riguardo bisogna dire che le Alpi, grazie alla configurazione del territorio, sono state per secoli il rifugio di una minoranza, la comunità valdese, che era sorta a Lione, ma ne era dovuta fuggire a causa delle persecuzioni.
antecedent-mention: la comunità valdese-ne
tag: best_correct, true label: 0, pred: 0, conf: 1.00
-----------------
Sentence: Il matrimonio con Olga Khokhlova si concluse in una separazione anziché in un divorzio perché secondo le leggi francesi un divorzio avrebbe significato dividere equamente le proprietà della coppia tra i due coniugi, cosa che Picasso non volle fare.
antecedent-mention: una separazione-un divorzio
tag: best_correct, true label: 0, pred: 0, conf: 1.00
-----------------
Sentence: Nel 1928 va a New York, dove tiene varie mostre personali, realizza varie ambientazioni di ristoranti, crea scenografie e costumi per vari teatri e molte coreografie.
antecedent-mention: New York-va
tag: best_correct, true label: 0, pred: 0, conf: 1.00
-----------------
Sentence: Continuò gli studi presso l'Università di Padova e si laureò a Roma.
antecedent-mention: Continuò-Roma
tag: best_correct, true label: 0, pred: 0, conf: 1.00
-----------------
Sentence: Dominata inizialmente da Ezzelino da Romano, passò poi tra le mani degli Scaligeri (1262), che ampliarono notevolmente la città e i suoi domini.
antecedent-mention: Ezzelino da Romano-passò
tag: worst_incorrect, true label: 0, pred: 1, conf: 1.00
-----------------
Sentence: I Racconti d'Italia e di Spagna ci trasportarono in un romanticismo beffardo, che ci riposò, senza che ce ne rendessimo conto, del convinto romanticismo di Victor Hugo».
antecedent-mention: rendessimo-ci
tag: worst_incorrect, true label: 1, pred: 0, conf: 1.00
-----------------
Sentence: Come sede del museo da loro ideato affittarono un edificio piuttosto modesto e lo aprirono al pubblico il 7 novembre 1929, 9 giorni dopo il crollo di Wall Street.
antecedent-mention: affittarono-un edificio piuttosto modesto
tag: worst_incorrect, true label: 0, pred: 1, conf: 1.00
-----------------
Sentence: Il 7 dicembre 1943, si consacrò con voti privati a Dio, scegliendolo come il''Tutto''della sua vita.
antecedent-mention: Dio-scegliendolo
tag: worst_incorrect, true label: 1, pred: 0, conf: 1.00
-----------------
Sentence: L'autostrada del Brennenero (A 22) taglia longitudinalmente da est ad ovest l'intero territorio.
antecedent-mention: L'autostrada del Brennenero (A 22-A 22)
tag: worst_incorrect, true label: 1, pred: 0, conf: 1.00
-----------------
Sentence: Un caro amico conosciuto a Pavia gli propose di acquistare una grande farmacia in Lombardia abbandonando lo sconosciuto paesello della provincia trentina, ma Nino Scaglia cantò così il suo amore per Storo e il suo orgoglio di essere storese:
antecedent-mention: lo sconosciuto paesello della provincia trentina-Storo
tag: lowconf_correct, true label: 1, pred: 1, conf: 0.50
-----------------
Sentence: se questo è vero soprattutto nel mondo occidentale, è anche vero che le esplorazioni delle radici musicali africane che molti jazzisti intrapresero a partire dagli anni sessanta e i contatti tra culture e stili musicali caratteristici dell'ultima parte del XX secolo, hanno contribuito a creare molti tipi di jazz, che vanno dalla tradizionale performance per piccolo ensemble, derivato dalle esperienze boppistiche e post - boppistiche, alla creazione di sonorità insolite che nascono dalla ibridazione di diverse tradizioni strumentali e musicali, fino ad arrivare a dissolversi nel genere chiamato world music (e in questo caso non si parla più di jazz).
antecedent-mention: dissolversi-jazz
tag: lowconf_correct, true label: 0, pred: 0, conf: 0.51
-----------------
Sentence: Queste preziose notizie le apprendiamo dalle relazioni che il provveditore di Salò, Daniele Dandolo, fece a Venezia a riguardo dell'impresa compiuta dal''connestabile''Scipione degli Ugoni, difatti amaramente soggiunge:
antecedent-mention: Queste preziose notizie-le
tag: lowconf_correct, true label: 1, pred: 1, conf: 0.52
-----------------
Sentence: Il Giro assunse sempre più importanza ed iniziarono a parteciparvi anche team stranieri.
antecedent-mention: Il Giro-parteciparvi
tag: lowconf_correct, true label: 1, pred: 1, conf: 0.53
-----------------
Sentence: Nel 1987 la gara tornò nel suo usuale formato e le venne riconosciuto lo status di " internazionale ".
antecedent-mention: la gara-le
tag: lowconf_correct, true label: 1, pred: 1, conf: 0.54
-----------------


Model name: Musixmatch/umberto-wikipedia-uncased-v1

Sentence: Alpi innevate viste dal satellite (13 marzo 2007) Le Alpi fanno parte delle catene alpine peritetidee, formatesi durante il Mesozoico e il Cenozoico, che si estendono dal Maghreb all'Estremo Oriente.
antecedent-mention: delle catene alpine peritetidee-il Cenozoico
tag: best_correct, true label: 0, pred: 0, conf: 1.00
-----------------
Sentence: Struttura geografico - morfologica e storia hanno fatto sì che il territorio trentino si presenti, a grandi linee, ripartito in tre grosse aree dialettologiche: quella occidentale, contraddistinta da forti influssi del dialetto lombardo, quella orientale, caratterizzata invece da influenze venetizzanti e quella centrale, che un tempo era più simile alle valli occidentali, oggi in avanzata fase di venetizzazione (Bonfadini 1983, p. 45).
antecedent-mention: quella centrale-1983
tag: best_correct, true label: 0, pred: 0, conf: 1.00
-----------------
Sentence: Castel Selva, le cui rovine dominano l'omonimo abitato, risale al XII secolo, durante il XVI secolo fu decorato in maniera sontuosa da Bernardo Clesio, che ne fece la sua residenza estiva, e proprio per questo durante il Concilio di Trento fu residenza di molti emissari giunti per assistere ai lavori, tra cui Marcello Corvini (futuro papa Marcello II) e Reginald Pole, arcivescovo di Canterbury assieme al principe vescovo Cristoforo Madruzzo e al segretario del Concilio, Massarello.
antecedent-mention: le cui rovine-molti emissari giunti per assistere ai lavori
tag: best_correct, true label: 0, pred: 0, conf: 1.00
-----------------
Sentence: Decisivo per il pittore fu un suo viaggio a Tunisi e ad Hammamet con Louis Moilliet e Lacke nel 1914.
antecedent-mention: il pittore-Tunisi
tag: best_correct, true label: 0, pred: 0, conf: 1.00
-----------------
Sentence: Ma anche a questo riguardo bisogna dire che le Alpi, grazie alla configurazione del territorio, sono state per secoli il rifugio di una minoranza, la comunità valdese, che era sorta a Lione, ma ne era dovuta fuggire a causa delle persecuzioni.
antecedent-mention: ne-Lione
tag: worst_incorrect, true label: 1, pred: 0, conf: 1.00
-----------------
Sentence: Nel 1998 la rivista Boss Magazine, stando a quanto affermato dalla Merz senza accordi precedenti, realizza un calendario sexy per l'anno 1999 utilizzando le foto di un servizio che aveva effettuato per la rivista [1] e nel 1999 viene scelta per il calendario 2000 della rivista Maxim.
antecedent-mention: dalla Merz-nel 1999
tag: worst_incorrect, true label: 1, pred: 0, conf: 1.00
-----------------
Sentence: I grandi successi conseguiti gli valsero la nomina da parte dell'Imperatore a Cavaliere di Moldelba (Ritter von Moldelbe) (20 ottobre 1850).
antecedent-mention: Moldelba-Moldelbe
tag: worst_incorrect, true label: 1, pred: 0, conf: 1.00
-----------------
Sentence: il tipo di alpeggio in genere nel periodo della monticazione avviene il passaggio del bestiame in diverse strutture d'alta quota.
antecedent-mention: alpeggio-della monticazione
tag: worst_incorrect, true label: 1, pred: 0, conf: 1.00
-----------------
Sentence: Vincent Van Gogh compì una svolta proprio grazie agli impressionisti, ma da loro si discostò, precorrendo l'Espressionismo.
antecedent-mention: agli impressionisti-discostò
tag: worst_incorrect, true label: 0, pred: 1, conf: 1.00
-----------------
Sentence: I due rimasero insieme fino alla morte dell'artista, sposandosi nel 1961.
antecedent-mention: dell'artista-nel 1961
tag: lowconf_correct, true label: 1, pred: 1, conf: 0.50
-----------------
Sentence: Nel 1905 entrò a far parte della redazione del giornale Il Trentino e in breve tempo assunse la carica di direttore, scrisse una serie di articoli con cui difendeva l'italianità e l'autonomia culturale del Trentino dai tentativi di germanizzazione proposti dalle forze politiche nazionaliste del Tirolo tedesco, ma non mise in discussione l'appartenenza all'Impero Austro - Ungarico.
antecedent-mention: una serie di articoli con cui-mise
tag: lowconf_correct, true label: 0, pred: 0, conf: 0.51
-----------------
Sentence: La successiva presenza longobarda è citata in vari documenti e, anch'essa, in toponimi quali Guizza (da wizza,''pascolo della comunità'').
antecedent-mention: La successiva presenza longobarda-essa
tag: lowconf_correct, true label: 1, pred: 1, conf: 0.51
-----------------
Sentence: La sede centrale dell'associazione si trova a Trento nel Palazzo Saracini - Cresseri (sec. XVI) che ospita oltre all'Organizzazione centrale, il Museo della SAT, l'Archivio storico, la Biblioteca della montagna, la Direzione provinciale del Corpo soccorso alpino S.A. T., la Sezione SAT di Trento, la S. U.S. A.T. (Sezione universitaria), il Coro della SAT, il Collegio provinciale delle Guide alpine, il Gruppo rocciatori SAT e la Scuola di alpinismo e sci alpinismo''Giorgio Graffer''.
antecedent-mention: dell'associazione-l'Archivio storico
tag: lowconf_correct, true label: 0, pred: 0, conf: 0.51
-----------------
Sentence: Ferdinand von Richthofen chiamò Martini''Il più grande geografo della missione Cinese, che non è stato uguagliato da nessun altro nel XVIII secolo SUS P. Non ci sono stati altri missionari, nè prima, nè dopo, che hanno fatto un uso così diligente del proprio tempo nell'acquisire informazioni riguardanti il paese.
antecedent-mention: altri missionari-proprio
tag: lowconf_correct, true label: 1, pred: 1, conf: 0.55
-----------------


Model name: idb-ita/gilberto-uncased-from-camembert

Sentence: Questo continuo scambio di persone e di esperienze, con il passare del tempo, contribuì a fare in modo che le leggi e la loro applicazione tendessero a diventare omogenee in città anche distanti tra loro, ma nelle quali avevano governato gli stessi podestà.
antecedent-mention: le leggi-loro
tag: best_correct, true label: 1, pred: 1, conf: 1.00
-----------------
Sentence: Alpi innevate viste dal satellite (13 marzo 2007) Le Alpi fanno parte delle catene alpine peritetidee, formatesi durante il Mesozoico e il Cenozoico, che si estendono dal Maghreb all'Estremo Oriente.
antecedent-mention: delle catene alpine peritetidee-il Cenozoico
tag: best_correct, true label: 0, pred: 0, conf: 1.00
-----------------
Sentence: Venne costruita perciò la via militare Claudia Augusta Altinate, che partiva da Altinum (attuale Altino, vicino a Venezia), attraversava la Valsugana, e passando per la piana di Pergine (ancora spoglia di edifici), arrivavava fino a Tridentum (attuale Trento).
antecedent-mention: la via militare Claudia Augusta Altinate-la Valsugana
tag: best_correct, true label: 0, pred: 0, conf: 1.00
-----------------
Sentence: A Storo ha sede l'Istituto comprensivo del Chiese " don Milani " con la scuola secondaria di primo grado (scuola media) di Storo e Pieve di Bono, e scuole primarie (scuole elementari) di Storo, Lodrone, Condino e Pieve di Bono per un totale di oltre 800 alunni.
antecedent-mention: Pieve di Bono-800 alunni
tag: best_correct, true label: 0, pred: 0, conf: 1.00
-----------------
Sentence: Seguirono una serie di scontri armati (assedio ed espugnazione del castello del'Mal Consigliò nel 1277 da parte dei trentini (che diventò allora'Buonconsigliò), a cui seguì la presa di Bolzano da parte di Mainardo (che ne rase al suolo le mura) che riuscì a prevalere.
antecedent-mention: ne-Bolzano
tag: worst_incorrect, true label: 1, pred: 0, conf: 1.00
-----------------
Sentence: L'alpeggio, che si svolge tra un'altitudine minima di 600 m s. l. m. e una massima di 2500 - 2700, inizia con la monticazione, cioè la salita sull'alpe, che avviene tra la fine di maggio e la metà di giugno e termina con la demonticazione, cioè la ridiscesa in pianura che avviene a fine settembre.
antecedent-mention: 2700-termina
tag: worst_incorrect, true label: 1, pred: 0, conf: 1.00
-----------------
Sentence: Iacopo (o Giacomo) Aconcio (Trento o Ossana, ca 1492 - Londra, ca 1567), fu un filosofo, un giurista, un riformatore religioso e un ingegnere italiano.
antecedent-mention: Ossana-1567
tag: worst_incorrect, true label: 1, pred: 0, conf: 1.00
-----------------
Sentence: Giorgio de Chirico (Volos, 10 luglio 1888 - Roma, 20 novembre 1978) è stato un pittore italiano, principale esponente della corrente artistica della pittura metafisica.
antecedent-mention: Roma-1978
tag: worst_incorrect, true label: 1, pred: 0, conf: 1.00
-----------------
Sentence: Il Concilio di Trento o Concilio Tridentino (svoltosi dal 1545 al 1563, con interruzioni) è considerato dalla Chiesa cattolica il XIX Concilio ecumenico.
antecedent-mention: al 1563-dalla Chiesa cattolica
tag: worst_incorrect, true label: 1, pred: 0, conf: 1.00
-----------------
Sentence: Avrebbe assorbito il progresso nella sua evoluzione e avrebbe dimostrato che il progresso era quello che era perché l'Uomo lo avrebbe usato per lasciar esplodere sinceramente la sua natura, che è fatta di istinto.
antecedent-mention: il progresso-lo
tag: lowconf_correct, true label: 1, pred: 1, conf: 0.51
-----------------
Sentence: Nel 1347 - 49 Pergine faceva ancora parte del principato Vescovile di Trento, ma i principi - vescovi non poterono prendere possesso di questa sede perchè i Tirolesi, vantando dei diritti, volevano appropriarsene.
antecedent-mention: Pergine-appropriarsene
tag: lowconf_correct, true label: 1, pred: 1, conf: 0.52
-----------------
Sentence: Comboni aveva un'illimitata fiducia nelle capacità dei popoli africani e si prefisse la fondazione di scuole in cui formare medici, insegnanti, preti e suore africani e da istituire in luoghi il cui clima fosse sopportabile anche per gli europei.
antecedent-mention: Comboni-prefisse
tag: lowconf_correct, true label: 1, pred: 1, conf: 0.52
-----------------
Sentence: Nel dicembre del 2004 la Saeco e la Lampre si fondono dando luogo alla Lampre - Caffita e così Simoni si trasferisce nella nuova squadra, assieme al suo rivale Damiano Cunego.
antecedent-mention: del 2004-la Lampre
tag: lowconf_correct, true label: 0, pred: 0, conf: 0.52
-----------------
Sentence: A Maloja, il villaggio alpino che lo ospitò dal 1894 fino alla morte, i luoghi che il pittore frequentava, verso i quali passeggiava e nei quali traeva l'ispirazione per le sue opere sono oggi uniti da un percorso commemorativo in 12 tappe, il Segantini Weg:
antecedent-mention: lo-passeggiava
tag: lowconf_correct, true label: 1, pred: 1, conf: 0.52
-----------------


Model name: m-polignano-uniba/bert_uncased_L-12_H-768_A-12_italian_alb3rt0

Sentence: Fin da ragazzo dimostra grande passione per l'arte figurativa convincendo i parenti a permettergli di iscriversi all'Accademia di Belle Arti di Bologna.
antecedent-mention: ragazzo-dimostra
tag: best_correct, true label: 0, pred: 0, conf: 1.00
-----------------
Sentence: La bandiera non ha origini antiche (mentre lo stemma si), deriva con molta probabilità dal tricolore francese, con i colori nella bandiera e nello stemma.
antecedent-mention: lo stemma-molta probabilità
tag: best_correct, true label: 0, pred: 0, conf: 1.00
-----------------
Sentence: In genere la polenta viene presentata in tavola su di un asse circolare coperto da uno straccio e viene servita, a seconda della sua consistenza, con un cucchiaio o a fette, queste ultime un tempo tagliate con un filo di cotone, dal basso verso l'alto.
antecedent-mention: fette-un tempo
tag: best_correct, true label: 0, pred: 0, conf: 1.00
-----------------
Sentence: Ciò che fu il limite della letteratura italiana alla fine dell'Ottocento, la sua mancanza di contenuti forti, il suo quieto e passivo laissez faire, venne immediatamente combattuto dai Futuristi (vedi art. 1, 2, 3) e la loro reazione comprese l'uso dell'eccesso, che provò l'esistenza di una sopravvissuta e dinamica classe intellettuale italiana.
antecedent-mention: vedi-l'uso dell'eccesso
tag: best_correct, true label: 0, pred: 0, conf: 1.00
-----------------
Sentence: Per quanto Cézanne accettasse l'impressionismo e ne condividesse gli obiettivi, non si identificava con esso e i suoi risultati sono infatti diversi:
antecedent-mention: ne-l'impressionismo
tag: worst_incorrect, true label: 1, pred: 0, conf: 1.00
-----------------
Sentence: I grandi successi conseguiti gli valsero la nomina da parte dell'Imperatore a Cavaliere di Moldelba (Ritter von Moldelbe) (20 ottobre 1850).
antecedent-mention: Moldelba-Moldelbe
tag: worst_incorrect, true label: 1, pred: 0, conf: 1.00
-----------------
Sentence: Il suo metodo di accertamento della verità è analitico, un metodo che intende svolgersi passando dalle proposizioni più generali alle più particolari, ritenendo che in questo modo, diversamente dal metodo sintetico, il soggetto conoscente resti legato ai dati dell'esperienza immediata, garantendosi una maggiore possibilità di appropriazione della verità.
antecedent-mention: Il suo metodo di accertamento della verità-garantendosi
tag: worst_incorrect, true label: 0, pred: 1, conf: 1.00
-----------------
Sentence: Nominato dall'Imperatore Francesco Giuseppe Cavaliere di Moldelba, nei Paesi di lingua tedesca, dove operò a lungo, è conosciuto come Alois Negrelli, Ritter von Moldelbe.
antecedent-mention: Moldelba-Moldelbe
tag: worst_incorrect, true label: 1, pred: 0, conf: 1.00
-----------------
Sentence: La musica che sarebbe stata chiamata " jass " e poco dopo " jazz " nasce quasi certamente a New Orleans all'inizio del XX secolo.
antecedent-mention: jass-jazz
tag: worst_incorrect, true label: 1, pred: 0, conf: 1.00
-----------------
Sentence: Il castello è di origine medievale e la sua posizione risulta particolarmente strategica, sia per l'ampio domino di terre che va dall'alta Valsugana, ai monti Celva e Calisio, alla vallata del Fersina e parte della Val dei Mocheni, sia per la posizione di collegamento tra Veneto e Trentino.
antecedent-mention: Il castello-terre
tag: lowconf_correct, true label: 0, pred: 0, conf: 0.51
-----------------
Sentence: Venne riconosciuta come ufficiale la versione della Bibbia detta Vulgata, evitando l'uso del volgare per le Sacre Scritture nel culto.
antecedent-mention: della Bibbia-le Sacre Scritture
tag: lowconf_correct, true label: 1, pred: 1, conf: 0.52
-----------------
Sentence: Le prime notizie certe sul maniero risalgono al 845, per poi ritrovare documenti datati nel XII secolo, il castello appartenne prima ai signori di Pergine, poi attraverso un susseguirsi di assedi e conquiste passo nelle mani dei duchi d'Austria sotto il regno di Margarete Maultasch e successivamente dell'imperatore Massimiliano I. Nel 1531 passò nelle mani dei Principi Vescovi di Trento, nella persona di Bernardo Clesio.
antecedent-mention: sul maniero-il castello
tag: lowconf_correct, true label: 1, pred: 1, conf: 0.53
-----------------
Sentence: Inoltre, mentre era in viaggio verso Roma, incontrò tipografi ad Anversa, Vienna e Monaco di Baviera per sottoporgli i suoi scritti storici e cartografici che aveva preparato.
antecedent-mention: era-aveva
tag: lowconf_correct, true label: 1, pred: 1, conf: 0.54
-----------------
Sentence: Prodi che lo volle come ministro della Difesa nel suo primo governo (maggio 1996 - ottobre 1998), un ruolo dove Andreatta si distinse per la forza delle sue proposte:
antecedent-mention: Prodi che-lo
tag: lowconf_correct, true label: 0, pred: 0, conf: 0.55
-----------------

## functions for calculating and displaying cosine similarity

import torch
import torch.nn.functional as F
import seaborn as sns
import matplotlib.pyplot as plt
import numpy as np


def find_token_indices(tokenised_sentence, tokenised_target):
    """
    Finds token index or indices in cases where the word was divided into subunits.
    """
    for i in range(len(tokenised_sentence) - len(tokenised_target) + 1):
        if tokenised_sentence[i:i+len(tokenised_target)] == tokenised_target:
            return list(range(i, i+len(tokenised_target)))
    return None


def compare_matrices(mat1, mat2, tokens1, tokens2, title1="Method 1", title2="Method 2", annot=True):
    """
    Plots two plots side by side for comparison.
    It gets two matrices as arguments, the tokens for the two matrices, the titles for the two matrices.
    """
    fig, axs = plt.subplots(1, 2, figsize=(16, 6))
    sns.heatmap(mat1, xticklabels=tokens1, yticklabels=tokens1, annot=annot, fmt=".2f", cmap="GnBu", ax=axs[0])
    axs[0].set_title(title1)

    sns.heatmap(mat2, xticklabels=tokens2, yticklabels=tokens2, annot=annot, fmt=".2f", cmap="GnBu", ax=axs[1])
    axs[1].set_title(title2)

    plt.tight_layout()
    plt.show()



def cosine_sim_matrix_single(tokenizer, model, sentence, show_plot=False, annot=True):
    """
    Takes a single sentence and gives a matrix graph with cosine similarity scores between all tokens (last hidden state).
    """
    model.eval()
    
    tokens_list = []
    
    # tokenise (return_tensors for batching)
    inputs = tokenizer(sentence.lower(), return_tensors="pt")
    tokens = tokenizer.convert_ids_to_tokens(inputs["input_ids"].squeeze())
    tokens_list = tokens[0:(len(tokens))] # add tokens to list (avoid special tokens); to include special tokens: tokens[0:(len(tokens))]

    # run model
    with torch.no_grad():
        outputs = model(**inputs)
    
    # get last hidden states
    last_hidden = outputs.last_hidden_state.squeeze(0)  # shape: (seq_len, hidden_dim)
    
    # initialise empty matrix 
    n = len(tokens_list)
    sim_matrix = np.zeros((n, n)) 
    
    for i, e1 in enumerate(tokens_list):
        for j, e2 in enumerate(tokens_list):
            # extract indices
            idx_1 = tokens.index(e1)
            idx_2 = tokens.index(e2)
            # extract embeddings
            vec_1 = last_hidden[idx_1]
            vec_2 = last_hidden[idx_2]
            # calculate cosine similarity
            sim_matrix[i][j] = F.cosine_similarity(vec_1, vec_2, dim=0).item()
    
    # plot heatmap
    if show_plot:    
        plt.figure(figsize=(10, 8))
        sns.heatmap(sim_matrix, xticklabels=tokens_list, yticklabels=tokens_list, annot=annot, fmt=".2f", cmap="Greens", vmin=0.0, vmax=1.0)
        plt.title("Cosine Similarity between Tokens")
        plt.show()
    
    return sim_matrix


def cosine_sim_across(tokenizer, model, sentence, word1, word2, show_plot=False):
    """
    Takes a single sentence and gives a graph showing cosine similarity through layers between two selected words.
    """
    model.eval()
    
    # tokenise (return_tensors for batching)
    inputs = tokenizer(sentence.lower(), return_tensors="pt")
    tokens = tokenizer.convert_ids_to_tokens(inputs["input_ids"].squeeze())
    
    # run model: get all hidden states (13 total: embedding + 12 layers)
    with torch.no_grad():
        outputs = model(**inputs)
        hidden_states = outputs.hidden_states  # tuple of len 13 for layers: 0 is embedding layer, 1-12 are transformer layers. each element has shape: (batch_size(1), seq_len, hidden_dim(768))
        # hidden_states[0] # gives embedding layer
        # hidden_states[12] # gives what .last_hidden_state gives 
    
    # locate the indices of word1 and word2
    tok1 = tokenizer.tokenize(word1.lower())
    tok2 = tokenizer.tokenize(word2.lower())
    idx_1 = find_token_indices(tokens,tok1)
    idx_2 = find_token_indices(tokens,tok2)
    
    # store scores across layers 
    similarities = []
    
    for layer in range(1, len(hidden_states)):  # skip layer 0 (embedding layer)
        layer_output = hidden_states[layer].squeeze(0)  # shape: (seq_len, hidden_dim)
        vec_1 = layer_output[idx_1].mean(dim=0)  # for multi-element words
        vec_2 = layer_output[idx_2].mean(dim=0)
        sim = F.cosine_similarity(vec_1, vec_2, dim=0).item()
        similarities.append(sim)

    # plot
    if show_plot:
        plt.plot(range(1, 13), similarities, marker='o')
        plt.ylim(-1.0, 1.0)
        plt.axhline(0.0, color='red', linestyle='--', linewidth=1)
        plt.xlabel(f"Model Layer ({model_name})")
        plt.ylabel(f"Cosine Similarity ('{word1}' vs. '{word2}')")
        plt.title("Cosine Similarity Across Layers")
        plt.xticks(range(1, 13))
        plt.grid(True)
        plt.show()
        
        print(f"Average cosine similarity between '{word1}' and '{word2}' (avg of per-layer similarities): {np.mean(similarities):.4f}")

    return similarities
        

def cosine_sim_matrix_average_sim(tokenizer, model, sentence, show_plot=False):
    """
    Takes a single sentence and gives a matrix graph with cosine similarity scores between all tokens averaged from all layers.
    ! It computes the similarities between tokens, THEN averages the similarities. !
    Returns a similarity matrix and tokens to be plotted side by side for comparison.
    """
    model.eval()
    
    tokens_list = []
    
    # tokenise (return_tensors for batching)
    inputs = tokenizer(sentence.lower(), return_tensors="pt")
    tokens = tokenizer.convert_ids_to_tokens(inputs["input_ids"].squeeze())
    tokens_list = tokens[0:(len(tokens))] # add tokens to list (avoid special tokens); to include special tokens: tokens[0:(len(tokens))]
    
    # run model: get all hidden states (13 total: embedding + 12 layers)
    with torch.no_grad():
        outputs = model(**inputs)
        hidden_states = outputs.hidden_states  # tuple of len 13 for layers: 0 is embedding layer, 1-12 are transformer layers. each element has shape: (batch_size(1), seq_len, hidden_dim(768))
        # hidden_states[0] # gives embedding layer
        # hidden_states[12] # gives what .last_hidden_state gives 
    
    # initialise empty matrix 
    n = len(tokens_list)
    sim_matrix = np.zeros((n, n)) 
    
    # compute cosine similarity per layer and average
    for i in range(n):
        for j in range(n):
            similarities = []
    
            for layer in range(1, len(hidden_states)):  # skip layer 0
                layer_output = hidden_states[layer].squeeze(0)  # shape: (seq_len, hidden_dim)
                vec_1 = layer_output[i]
                vec_2 = layer_output[j]
                sim = F.cosine_similarity(vec_1, vec_2, dim=0).item()
                similarities.append(sim)
    
            sim_matrix[i][j] = np.mean(similarities)
    
    # plot heatmap
    if show_plot:
        plt.figure(figsize=(10, 8))
        sns.heatmap(sim_matrix, xticklabels=tokens, yticklabels=tokens, annot=True, fmt=".2f", cmap="GnBu", vmin=0.0, vmax=1.0)
        plt.title("Cosine Similarity Between Tokens (Avg. of Per-Layer Similarities)")
        plt.show()

    return sim_matrix, tokens

    
def cosine_sim_matrix_average_emb(tokenizer, model, sentence, show_plot=False):
    """
    Takes a single sentence and gives a matrix graph with cosine similarity scores between all tokens averaged from all layers.
    ! It averages the embeddings across layers, THEN computes the similarity. !
    Returns a similarity matrix and tokens to be plotted side by side for comparison.
    """
    model.eval()
    
    tokens_list = []
    
    # tokenise (return_tensors for batching)
    inputs = tokenizer(sentence.lower(), return_tensors="pt")
    tokens = tokenizer.convert_ids_to_tokens(inputs["input_ids"].squeeze())
    tokens_list = tokens[0:(len(tokens))] # add tokens to list (avoid special tokens); to include special tokens: tokens[0:(len(tokens))]
    
    # run model: get all hidden states (13 total: embedding + 12 layers)
    with torch.no_grad():
        outputs = model(**inputs)
        hidden_states = outputs.hidden_states  # tuple of len 13 for layers: 0 is embedding layer, 1-12 are transformer layers. each element has shape: (batch_size(1), seq_len, hidden_dim(768))
    
    # stack layers 1–12 (ignore layer 0) 
    layer_stack = torch.stack(hidden_states[1:], dim=0)  # shape: (12, 1, seq_len, 768)
    # average across layers 
    mean_rep = layer_stack.mean(dim=0) # shape: (1, seq_len, hidden_dim)
    # remove batch dimension
    mean_rep = mean_rep.squeeze(0) # shape: (seq_len, hidden_dim)
    
    # initialise empty matrix 
    n = len(tokens_list)
    sim_matrix = np.zeros((n, n)) 
    
    for i in range(len(tokens_list)):
        for j in range(len(tokens_list)):
            vec_1 = mean_rep[i]
            vec_2 = mean_rep[j]
            sim = F.cosine_similarity(vec_1, vec_2, dim=0).item()
            sim_matrix[i][j] = sim
    
    # plot heatmap
    if show_plot:
        plt.figure(figsize=(10, 8))
        sns.heatmap(sim_matrix, xticklabels=tokens, yticklabels=tokens, annot=True, fmt=".2f", cmap="GnBu", vmin=0.0, vmax=1.0)
        plt.title("Cosine Similarity Between Tokens (Avg. of Layers' Embeddings)")
        plt.show()

    return sim_matrix, tokens


def cosine_sim_single_average_emb(tokenizer, model, sentence, word1, word2, layers=range(1, 13), print_result=False):
    """
    Takes a single sentence and gives average cosine similarity score between two words.
    ! It averages the two embeddings across layers, THEN computes the similarity. !
    """
    model.eval()
    
    inputs = tokenizer(sentence.lower(), return_tensors="pt")
    tokens = tokenizer.convert_ids_to_tokens(inputs["input_ids"].squeeze())
    
    with torch.no_grad():
        outputs = model(**inputs)
        hidden_states = outputs.hidden_states
    
    tok1 = tokenizer.tokenize(word1.lower())
    tok2 = tokenizer.tokenize(word2.lower())
    idx_1 = find_token_indices(tokens, tok1)
    idx_2 = find_token_indices(tokens, tok2)
    
    # average across selected layers
    vec_1 = torch.stack([hidden_states[i].squeeze(0)[idx_1].mean(dim=0) for i in layers]).mean(dim=0)
    vec_2 = torch.stack([hidden_states[i].squeeze(0)[idx_2].mean(dim=0) for i in layers]).mean(dim=0)
    
    sim = F.cosine_similarity(vec_1, vec_2, dim=0).item()

    if print_result:
        print(f"Average cosine similarity between '{word1}' and '{word2}': {sim}")
    
    return sim
    

def cosine_sim_single(tokenizer, model, sentence, word1, word2, print_result=False):
    """
    Takes a single sentence and gives cosine similarity score between two selected words (last layer).
    """
    model.eval()
        
    # tokenise (return_tensors for batching)
    inputs = tokenizer(sentence.lower(), return_tensors="pt")
    tokens = tokenizer.convert_ids_to_tokens(inputs["input_ids"].squeeze())

    #run model
    with torch.no_grad():
        outputs = model(**inputs)
    
    # get last hidden states
    last_hidden = outputs.last_hidden_state.squeeze(0)  # shape: (seq_len, hidden_dim)

    # extract indices
    tok1 = tokenizer.tokenize(word1.lower())
    tok2 = tokenizer.tokenize(word2.lower())
    idx_1 = find_token_indices(tokens,tok1)
    idx_2 = find_token_indices(tokens,tok2) 
    # extract embeddings
    vec_1 = last_hidden[idx_1].mean(dim=0) # for multi-element words
    vec_2 = last_hidden[idx_2].mean(dim=0)
    
    # compute cosine similarity
    similarity = F.cosine_similarity(vec_1, vec_2, dim=0).item()
    if print_result:
        print(f"Cosine similarity between '{word1}' and '{word2}' (last layer): {similarity:.4f}")
    return similarity

    
def cosine_sim_single_layer(tokenizer, model, sentence, word1, word2, layer, print_result=False):
    """
    Takes a single sentence and gives cosine similarity score between two selected words (selected layer).
    """
    model.eval()
    
    # tokenise (return_tensors for batching)
    inputs = tokenizer(sentence.lower(), return_tensors="pt")
    tokens = tokenizer.convert_ids_to_tokens(inputs["input_ids"].squeeze())
    
    # run model: get all hidden states (13 total: embedding + 12 layers)
    with torch.no_grad():
        outputs = model(**inputs)
        #hidden_states_layer = outputs.hidden_states[layer]  # tuple of len 13 for layers: 0 is embedding layer, 1-12 are transformer layers. each element has shape: (batch_size(1), seq_len, hidden_dim(768))
        # hidden_states[0] # gives embedding layer
        # hidden_states[12] # gives what .last_hidden_state gives 
    
    # extract indices
    tok1 = tokenizer.tokenize(word1.lower())
    tok2 = tokenizer.tokenize(word2.lower())
    idx_1 = find_token_indices(tokens,tok1)
    idx_2 = find_token_indices(tokens,tok2) 
    # extract embeddings
    layer_output = outputs.hidden_states[layer].squeeze(0)
    vec_1 = layer_output[idx_1].mean(dim=0) # for multi-element words
    vec_2 = layer_output[idx_2].mean(dim=0)
    
    # compute cosine similarity
    similarity = F.cosine_similarity(vec_1, vec_2, dim=0).item()
    if print_result:
        print(f"Cosine similarity between '{word1}' and '{word2}' in layer {layer}: {similarity:.4f}")
    return similarity

## function to draw graphs and statistics: boxplot, distribution bell, statistics
## plotted in an easy to read format (side by side for each model)

import matplotlib.pyplot as plt
import seaborn as sns
from scipy.stats import ttest_ind
import numpy as np
import pandas as pd

def plot_with_stats_below(df, sim_cols, model_name, label_spec):
    """
    Plots in a column: box-plot, KDE, statistics; it does so 3 times, one for each visualisation (last, layer 11, average).
    sim_cols is a list of tuples: (column_name, title_suffix)
    """
    column_name, val_1, val_2 = label_spec
    palette = {val_1: sns.color_palette()[1], val_2: sns.color_palette()[0]}
    n = len(sim_cols)
    #fig, axes = plt.subplots(3, n, figsize=(5 * n, 10), constrained_layout=True)
    fig, axes = plt.subplots(
        3, n,
        figsize=(5 * n, 10),
        constrained_layout=True,
        gridspec_kw={'height_ratios': [1.5, 1.5, 1]} 
    )

    for idx, (col, suffix) in enumerate(sim_cols):
        # subsets
        trueM = df[df[column_name] == val_1][col].dropna() # dropping examples which could not be worked
        falseM = df[df[column_name] == val_2][col].dropna()

        # stats
        mean_true, med_true = trueM.mean(), trueM.median()
        mean_false, med_false = falseM.mean(), falseM.median()
        if len(trueM) >= 2 and len(falseM) >= 2:
            t_stat, p_val = ttest_ind(trueM, falseM, nan_policy='omit', equal_var=False) # equal_var=False : Welch's t-test for groups with unequal variances
            mean_diff = mean_true - mean_false
            pooled_std = np.sqrt((trueM.std() ** 2 + falseM.std() ** 2) / 2)
            cohens_d = mean_diff / pooled_std if pooled_std != 0 else np.nan
        else:
            t_stat, p_val, cohens_d = np.nan, np.nan, np.nan

        vmin, vmax = -0.3, 1.25 # keep min and max values fixed for all models and plots
        # row 1: boxplot
        ax_box = axes[0, idx]
        sns.boxplot(
            data=df,
            x=column_name,
            y=col,
            palette=palette,
            order=[val_2, val_1],
            ax=ax_box,
        )
        ax_box.set_title(f"{model_name} — {suffix}", fontsize=12)
        ax_box.set_xlabel("")
        ax_box.set_ylim(vmin, vmax)
        ax_box.set_ylabel("Cosine Similarity")

        # row 2: KDE
        ax_kde = axes[1, idx]
        sns.kdeplot(data=df[df[column_name] == val_2], x=col,
                    label=val_2, fill=True, ax=ax_kde)
        sns.kdeplot(data=df[df[column_name] == val_1], x=col,
                    label=val_1, fill=True, ax=ax_kde)
        ax_kde.set_xlabel("Cosine Similarity")
        ax_kde.set_ylabel("Density")
        ax_kde.set_xlim(vmin, vmax)
        ax_kde.legend(fontsize=8)

        # row 3: summary stats as a mini table (render via text)
        ax_stats = axes[2, idx]
        ax_stats.axis("off")
        stats_df = pd.DataFrame({
            "mean": [mean_true, mean_false],
            "median": [med_true, med_false]
            },
            index=[val_1, val_2]
        ).round(3)
        p_str = f"p < 0.001" if p_val < 0.001 else f"p = {p_val:.3f}"
        stats_text = (
            f"{stats_df.to_string()}\n"
            f"t = {t_stat:.2f}, {p_str}\n"
            f"Cohen's d = {cohens_d:.2f}"
        )
        ax_stats.text(0, 0.5, stats_text, va="center", ha="left", fontsize=12, family="monospace")

    fig.suptitle(f"Cosine Similarity Distributions + Summary for {model_name}", fontsize=16, y=1.02)
    plt.show()

""" 
!! this is really long to run and crashes sometimes (because of unstable connection I think)
    it took ~15min per column per model, i.e., 3-col * 4-mod = ~3h
I run it for one model at a time and always saved the output so that if it crashed damages were contained
After running it I saved the dataframe locally with the added columns for cosine similarity
so the only needed step is to load the pre-processed dataframe at the beginning of this notebook
"""

## compute similarity across the dataset (add columns with similarity to dataset)
## i) last layer, ii) selected layer (11), iii) average 

from tqdm import tqdm

for m in model_pairs[1:]:
    model_name = m[0]
    model = AutoModel.from_pretrained(model_name, output_hidden_states=True)
    tokenizer = m[1]
    simple_name = m[3]
    
    print(f"Model name: {model_name}")
    
    # optional: if using CUDA
    #model = model.to("cuda")


    # i) last layer 
    coref_df_tf[f'similarity-lst-{simple_name}'] = 0.0
    skipped = 0
    
    for i, row in tqdm(coref_df_tf.iterrows(), total=len(coref_df_tf), desc="Computing similarities: last layer"):
        sentence = row['text']
        word1 = row['antecedent']
        word2 = row['mention']
    
        try:
            sim = cosine_sim_single(tokenizer, model, sentence, word1, word2)
        except Exception as e:
            sim = None  
            #print(f"Error at row {i}: {e}")
            skipped+=1
    
        coref_df_tf.at[i, f'similarity-lst-{simple_name}'] = sim
    print(f"Skipped examples: {skipped}")


    # ii) layer 11
    coref_df_tf[f'similarity-l11-{simple_name}'] = 0.0
    skipped = 0
    
    for i, row in tqdm(coref_df_tf.iterrows(), total=len(coref_df_tf), desc="Computing similarities: layer 11"):
        sentence = row['text']
        word1 = row['antecedent']
        word2 = row['mention']
    
        try:
            sim = cosine_sim_single_layer(tokenizer, model, sentence, word1, word2, 11)
        except Exception as e:
            sim = None  
            #print(f"Error at row {i}: {e}")
            skipped+=1
    
        coref_df_tf.at[i, f'similarity-l11-{simple_name}'] = sim
    print(f"Skipped examples: {skipped}")


    # iii) average (average embedding)
    coref_df_tf[f'similarity-avg-{simple_name}'] = 0.0
    skipped = 0
    
    for i, row in tqdm(coref_df_tf.iterrows(), total=len(coref_df_tf), desc="Computing similarities: average"):
        sentence = row['text']
        word1 = row['antecedent']
        word2 = row['mention']
    
        try:
            sim = cosine_sim_single_average_emb(tokenizer, model, sentence, word1, word2)
        except Exception as e:
            sim = None  
            #print(f"Error at row {i}: {e}")
            skipped+=1
    
        coref_df_tf.at[i, f'similarity-avg-{simple_name}'] = sim
    print(f"Skipped examples: {skipped}\n")

## visualisation of cosine similarity for all models: last layer, layer 11, average

## True vs False mentions

import warnings
warnings.filterwarnings("ignore", message="use_inf_as_na option is deprecated")

for m in model_pairs[1:]:
    simple_name = m[3]

    sim_cols = [
        (f"similarity-lst-{simple_name}", "Last layer"),
        (f"similarity-l11-{simple_name}", "Layer 11"),
        (f"similarity-avg-{simple_name}", "Average"),
    ]
    
    plot_with_stats_below(coref_df_tf, sim_cols, simple_name, ('label_str','True-mention','False-mention'))

## visualisation of cosine similarity for all models: last layer, layer 11, average

## Non-Dropped vs Dropped mentions in True mentions examples

true_df = coref_df_tf[coref_df_tf['label'] == 1]


## visualisation of cosine similarity for all models

import warnings
warnings.filterwarnings("ignore", message="use_inf_as_na option is deprecated")

for m in model_pairs[1:]:
    simple_name = m[3]

    sim_cols = [
        (f"similarity-lst-{simple_name}", "Last layer"),
        (f"similarity-l11-{simple_name}", "Layer 11"),
        (f"similarity-avg-{simple_name}", "Average"),
    ]
    
    plot_with_stats_below(true_df, sim_cols, simple_name, ('dropped_subject_str','Non-Dropped','Dropped'))

## visualisation of cosine similarity for all models: last layer, layer 11, average

## True Dropped mentions and False Dropped mentions

dropped_df = coref_df_tf[coref_df_tf['dropped_subject'] == 1]

## visualisation of cosine similarity for all models

import warnings
warnings.filterwarnings("ignore", message="use_inf_as_na option is deprecated")

for m in model_pairs[1:]:
    simple_name = m[3]

    sim_cols = [
        (f"similarity-lst-{simple_name}", "Last layer"),
        (f"similarity-l11-{simple_name}", "Layer 11"),
        (f"similarity-avg-{simple_name}", "Average"),
    ]
    
    plot_with_stats_below(dropped_df, sim_cols, simple_name, ('label_str','True-mention','False-mention'))

## function get representations: tokenise single example, get attention (from layer range)
## function extract features

from sklearn.metrics import ConfusionMatrixDisplay, confusion_matrix
import numpy as np
import matplotlib.pyplot as plt

def plot_confusion_matrix(y_preds, y_true,labels):
    cm = confusion_matrix(y_true, y_preds, normalize="true")
    fig, ax = plt.subplots(figsize=(2, 2))
    disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=["0","1"])
    disp.plot(cmap="Greens", values_format=".2f", ax=ax, colorbar=False)
    plt.title("Normalized confusion matrix")
    plt.show()

def get_reps(example, model, tokenizer, layer_range):
    text = example['text']
    men_offset = example['mention_offset']
    ant_offset = example['antecedent_offset']

    enc = tokenizer(text, return_offsets_mapping=True, return_tensors="pt")
    ids = enc['input_ids']
    offsets = enc['offset_mapping'][0]

    with torch.no_grad():
        reps = model(**{k: v for k, v in enc.items() if k != 'offset_mapping'})
        hidden_states = reps.hidden_states  # tuple of [layer][batch][tokens][hidden]

    # average over selected layers
    selected = hidden_states[layer_range[0]:layer_range[1]]
    hidden = torch.stack(selected).mean(dim=0)[0]  # [tokens, hidden_size]

    def find_rep(char_offset):
        idx = next(i for i, (s, e) in enumerate(offsets) if s <= char_offset < e)
        return hidden[idx]

    men = find_rep(men_offset)
    ant = find_rep(ant_offset)

    return {
        'men': men,
        'ant': ant,
        'lab': int(example['label'])
    }



def extract_features(dataset, model, tokenizer, layer_range):
    features, labels = [], []
    for ex in tqdm(dataset, desc="Extracting features"):
        reps = get_reps(ex, model, tokenizer, layer_range)
        men = reps['men']
        ant = reps['ant']
        lab = reps['lab']
        features.append(torch.cat([men, ant]).numpy())
        labels.append(lab)

    return features, labels

"""
this is very long to run, with the accelerator on it took ~7 minutes per classifier: 
    i.e., 7 min * 4 layer-groups * 4 models = 112 minutes (~2h)
I trained each model's classifiers independently and stored results in a dataframe which I then downloaded locally
eventually I collected all results in a single dataframe and loaded it in order to plot the results
"""

## probe classifier on representations of first 3, middle 3a, middle 3b, last 3 layers 

from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
from sklearn.metrics import f1_score
from tqdm import tqdm
import torch

layer_groups = {
    "First 3": (0, 3),
    "Middle 3a": (3, 6),
    "Middle 3b": (6, 9),
    "Last 4": (9, 12)
}

x_pos = [3, 6, 9, 12]
all_model_results = {}
summary_report = {}
per_model_examples = {}



for m in model_pairs[4:5]:

    model_name = m[0]
    print(f"\n\n=== Evaluating model: {model_name} ===")
    tokenizer = m[1]
    model = AutoModel.from_pretrained(model_name, output_hidden_states=True)

    num_layers = model.config.num_hidden_layers
    num_heads = model.config.num_attention_heads

    model.eval()

    accuracies = []

    for group_name, layer_range in layer_groups.items():
        print(f"\n--- Layer Group: {group_name} ---")

        features, labels = extract_features(coref_dataset, model, tokenizer, layer_range)
        all_indices = np.arange(len(coref_dataset))
        X_train, X_test, y_train, y_test, idx_train, idx_test = train_test_split(
            features, labels, all_indices, test_size=0.2, random_state=42
        )

        clf = make_pipeline(StandardScaler(), LogisticRegression(max_iter=2000, solver='lbfgs'))
        clf.fit(X_train, y_train)
        y_preds = clf.predict(X_test)

        # Accuracy logging
        report = classification_report(y_test, y_preds, output_dict=True)
        accuracies.append(report['accuracy'])
        #print(classification_report(y_test, y_preds))
        f1 = f1_score(y_test, y_preds, average='macro')
        print(f"Macro F1 Score: {f1}")

        # Store best/worst examples
        if hasattr(clf, "predict_proba"):
            probs = clf.predict_proba(X_test)
            confidences = np.max(probs, axis=1)
        else:
            decision = clf.decision_function(X_test)
            confidences = 1 / (1 + np.exp(-np.abs(decision)))

        correct_mask = (y_preds == y_test)
        incorrect_mask = ~correct_mask

        best_correct = sorted(np.where(correct_mask)[0], key=lambda i: confidences[i], reverse=True)[:5]
        worst_incorrect = sorted(np.where(incorrect_mask)[0], key=lambda i: confidences[i], reverse=True)[:5]
        lowconf_correct = sorted(np.where(correct_mask)[0], key=lambda i: abs(confidences[i] - 0.5))[:5]

        per_model_examples[(model_name, group_name)] = {
            "best_correct": best_correct,
            "worst_incorrect": worst_incorrect,
            "lowconf_correct": lowconf_correct,
            "y_test": y_test,
            "y_preds": y_preds,
            "confidences": confidences,
            "X_test": X_test,
            "idx_test": idx_test
        }

        summary_report[(model_name, group_name)] = report

        #print(f"Correct: {sum(correct_mask)} | Incorrect: {sum(incorrect_mask)}")
        #plot_confusion_matrix(y_preds, y_test, labels=[0, 1])
        #print("-" * 60)

    all_model_results[model_name] = accuracies

## load dataframe with results from evaluation of models' probing classifiers

import pandas as pd

df = pd.read_csv("/kaggle/input/models-performance-3layergroups/model_layer_results.csv")

## print models' results 

model_name = None

for _, row in df.iterrows():
    current_model = row['model']
    
    if current_model != model_name:
        print(f"\nModel: {current_model}")
        model_name = current_model

    print(f"  Layer group: {row['layer_group']}   \tAccuracy: {row['accuracy']:.4f}")

Model: Musixmatch/umberto-commoncrawl-cased-v1
  Layer group: First 3   	Accuracy: 0.6714
  Layer group: Middle 3a   	Accuracy: 0.6866
  Layer group: Middle 3b   	Accuracy: 0.7242
  Layer group: Last 4   	Accuracy: 0.7242

Model: Musixmatch/umberto-wikipedia-uncased-v1
  Layer group: First 3   	Accuracy: 0.6796
  Layer group: Middle 3a   	Accuracy: 0.6972
  Layer group: Middle 3b   	Accuracy: 0.6831
  Layer group: Last 4   	Accuracy: 0.7089

Model: idb-ita/gilberto-uncased-from-camembert
  Layer group: First 3   	Accuracy: 0.6819
  Layer group: Middle 3a   	Accuracy: 0.7300
  Layer group: Middle 3b   	Accuracy: 0.6937
  Layer group: Last 4   	Accuracy: 0.7207

Model: m-polignano-uniba/bert_uncased_L-12_H-768_A-12_italian_alb3rt0
  Layer group: First 3   	Accuracy: 0.6538
  Layer group: Middle 3a   	Accuracy: 0.6960
  Layer group: Middle 3b   	Accuracy: 0.6960
  Layer group: Last 4   	Accuracy: 0.6749

## plot results

import matplotlib.pyplot as plt

# map layer group names to x-axis positions
layer_group_to_x = {
    "First 3": 3,
    "Middle 3a": 6,
    "Middle 3b": 9,
    "Last 4": 12
}
# new column for x-axis values
df["x"] = df["layer_group"].map(layer_group_to_x)


plt.figure(figsize=(10, 6))

for model_name in df["model"].unique():
    model_data = df[df["model"] == model_name].sort_values("x")
    plt.plot(model_data["x"], model_data["accuracy"], marker='o', label=model_name)

plt.xticks(ticks=range(0, 13))
plt.xlabel("Layer")
plt.ylabel("Accuracy")
plt.title("Layer-wise Probing Accuracy Across Models")
plt.grid(True)
plt.legend()
plt.tight_layout()
plt.show()

	token_id	sentence_id	word	ph1	lemma	ph2	pos	ph3	morpho_features	ph4	dep_head_id	ph5	dep_rel	onto_type	ph6	ph7	ph8	coref_col1	coref_col2	coref_chain_info
0	1	0	Castel	_	castel	_	SPN	_	_null_	_	0	_	ROOT	(facility	_	_	_	B-M_1=set_2=nominal=facility=B-MIN	B-M_1=set_2=nominal=facility=B-MIN	(1
1	2	0	Ivano	_	ivano	_	SPN	_	_null_	_	1	_	CONTIN-DENOM	facility)	_	_	_	I-M_1=set_2=nominal=facility=I-MIN	I-M_1=set_2=nominal=facility=I-MIN	1)
2	1	1	Castel	_	castel	_	SPN	_	_null_	_	3	_	SUBJ	(facility	_	_	_	B-M_3=set_2=nominal=facility=B-MIN	B-M_3=set_2=nominal=facility=B-MIN	(1
3	2	1	Ivano	_	ivano	_	SPN	_	_null_	_	1	_	CONTIN-DENOM	facility)	_	_	_	I-M_3=set_2=nominal=facility=I-MIN	I-M_3=set_2=nominal=facility=I-MIN	1)
4	3	1	è	_	essere	_	VI	_	essere+v+indic+pres+nil+3+sing	_	0	_	ROOT	_	_	_	_	O	O	_
5	4	1	una	_	un	_	RS	_	indet+art+f+sing	_	5	_	DET	_	_	_	_	B-M_5=set_29=nominal=null=B-MIN	B-M_5=set_30=nominal=null=B-MIN	(2
6	5	1	fortificazione	_	fortificazione	_	SS	_	fortificazione+n+f+sing	_	3	_	PRED	_	_	_	_	I-M_5=set_29=nominal=null=I-MIN	I-M_5=set_30=nominal=null=I-MIN	_
7	6	1	che	_	che	_	CCHE	_	che+conj	_	8	_	SUBJ	_	_	_	_	I-M_5=set_29=nominal=null=I-MIN	I-M_5=set_30=nominal=null=I-MIN	_
8	7	1	si	_	si	_	PN	_	rifl+pron+accdat+_+3+_+clit	_	8	_	DOBJ	_	_	_	_	I-M_5=set_29=nominal=null=I-MIN	I-M_5=set_30=nominal=null=I-MIN	_
9	8	1	erge	_	ergere	_	VI	_	ergere+v+indic+pres+nil+3+sing	_	5	_	RELCL	_	_	_	_	I-M_5=set_29=nominal=null=I-MIN	I-M_5=set_30=nominal=null=I-MIN	_

	text	antecedent	mention	dropped_subject	label
0	Ritenuto famoso per il rarissimo esempio in re...	appare	presenta	True	1
1	Ritenuto famoso per il rarissimo esempio in re...	appare	regione	False	0
2	Ritenuto famoso per il rarissimo esempio in re...	appare	il rarissimo esempio in regione di cammino di ...	False	0
3	Il primo documento storico è datato 1187 e rig...	Il primo documento storico	riguarda	True	1
4	Il primo documento storico è datato 1187 e rig...	Il primo documento storico	il Signore di Ivano	False	0

Lorenzo Severini:
Attention analysis and cosine similarity analysis of present and dropped mentions in Italian coreference dataset

Italian Language Models

Italian coreference dataset

Live Memories Corpus

Processing dataset

Processing of dataset (summary)

Attention visualisation

Visualisation of attention through attention matrices

Visualisation of attention through attention matrices (summary)

Training a probing classifier

Cosine similarity

Cosine similarity (summary)

Final experiment

Summary

Dataset¶

Models¶

Processing of dataset¶

Attention matrices¶

Probing classifier¶

Cosine similarity¶

Final experiment¶

Final observations¶

	model	layer_group	accuracy	x
0	Musixmatch/umberto-commoncrawl-cased-v1	First 3	0.671362	3
1	Musixmatch/umberto-commoncrawl-cased-v1	Middle 3a	0.686620	6
2	Musixmatch/umberto-commoncrawl-cased-v1	Middle 3b	0.724178	9
3	Musixmatch/umberto-commoncrawl-cased-v1	Last 4	0.724178	12
4	Musixmatch/umberto-wikipedia-uncased-v1	First 3	0.679577	3
5	Musixmatch/umberto-wikipedia-uncased-v1	Middle 3a	0.697183	6
6	Musixmatch/umberto-wikipedia-uncased-v1	Middle 3b	0.683099	9
7	Musixmatch/umberto-wikipedia-uncased-v1	Last 4	0.708920	12
8	idb-ita/gilberto-uncased-from-camembert	First 3	0.681925	3
9	idb-ita/gilberto-uncased-from-camembert	Middle 3a	0.730047	6
10	idb-ita/gilberto-uncased-from-camembert	Middle 3b	0.693662	9
11	idb-ita/gilberto-uncased-from-camembert	Last 4	0.720657	12
12	m-polignano-uniba/bert_uncased_L-12_H-768_A-12...	First 3	0.653756	3
13	m-polignano-uniba/bert_uncased_L-12_H-768_A-12...	Middle 3a	0.696009	6
14	m-polignano-uniba/bert_uncased_L-12_H-768_A-12...	Middle 3b	0.696009	9
15	m-polignano-uniba/bert_uncased_L-12_H-768_A-12...	Last 4	0.674883	12

Lorenzo Severini: Attention analysis and cosine similarity analysis of present and dropped mentions in Italian coreference dataset

Italian Language Models

Italian coreference dataset

Live Memories Corpus

Processing dataset

Processing of dataset (summary)

Attention visualisation

Visualisation of attention through attention matrices

Visualisation of attention through attention matrices (summary)

Training a probing classifier

Cosine similarity

Cosine similarity (summary)

Final experiment

Summary

Dataset¶

Models¶

Processing of dataset¶

Attention matrices¶

Probing classifier¶

Cosine similarity¶

Final experiment¶

Final observations¶

Lorenzo Severini:
Attention analysis and cosine similarity analysis of present and dropped mentions in Italian coreference dataset